diff --git a/.gitea/workflows/fixturenet-eth-plugeth-test.yml b/.gitea/workflows/fixturenet-eth-plugeth-test.yml index ced0823f..739e33b0 100644 --- a/.gitea/workflows/fixturenet-eth-plugeth-test.yml +++ b/.gitea/workflows/fixturenet-eth-plugeth-test.yml @@ -2,7 +2,10 @@ name: Fixturenet-Eth-Plugeth-Test on: push: - branches: 'ci-test' + branches: '*' + paths: + - '!**' + - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test' # Needed until we can incorporate docker startup into the executor container env: @@ -16,8 +19,16 @@ jobs: steps: - name: "Clone project repository" uses: actions/checkout@v3 - - name: "Install Python" - uses: cerc-io/setup-python@v4 + # At present the stock setup-python action fails on Linux/aarch64 + # Conditional steps below workaroud this by using deadsnakes for that case only + - name: "Install Python for ARM on Linux" + if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }} + uses: deadsnakes/action@v3.0.1 + with: + python-version: '3.8' + - name: "Install Python cases other than ARM on Linux" + if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }} + uses: actions/setup-python@v4 with: python-version: '3.8' - name: "Print Python version" diff --git a/.gitea/workflows/fixturenet-eth-test.yml b/.gitea/workflows/fixturenet-eth-test.yml index 5a0410f2..73804df5 100644 --- a/.gitea/workflows/fixturenet-eth-test.yml +++ b/.gitea/workflows/fixturenet-eth-test.yml @@ -2,7 +2,10 @@ name: Fixturenet-Eth-Test on: push: - branches: 'ci-test' + branches: '*' + paths: + - '!**' + - '.gitea/workflows/triggers/fixturenet-eth-test' # Needed until we can incorporate docker startup into the executor container env: @@ -16,7 +19,15 @@ jobs: steps: - name: "Clone project repository" uses: actions/checkout@v3 - - name: "Install Python" + # At present the stock setup-python action fails on Linux/aarch64 + # Conditional steps below workaroud this by using deadsnakes for that case only + - name: "Install Python for ARM on Linux" + if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }} + uses: deadsnakes/action@v3.0.1 + with: + python-version: '3.8' + - name: "Install Python cases other than ARM on Linux" + if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }} uses: actions/setup-python@v4 with: python-version: '3.8' diff --git a/.gitea/workflows/fixturenet-laconicd-test.yml b/.gitea/workflows/fixturenet-laconicd-test.yml new file mode 100644 index 00000000..ac397dad --- /dev/null +++ b/.gitea/workflows/fixturenet-laconicd-test.yml @@ -0,0 +1,55 @@ +name: Fixturenet-Laconicd-Test + +on: + push: + branches: '*' + paths: + - '!**' + - '.gitea/workflows/triggers/fixturenet-laconicd-test' + +# Needed until we can incorporate docker startup into the executor container +env: + DOCKER_HOST: unix:///var/run/dind.sock + + +jobs: + test: + name: "Run an Laconicd fixturenet test" + runs-on: ubuntu-latest + steps: + - name: 'Update' + run: apt-get update + - name: 'Setup jq' + run: apt-get install jq -y + - name: 'Check jq' + run: | + which jq + jq --version + - name: "Clone project repository" + uses: actions/checkout@v3 + # At present the stock setup-python action fails on Linux/aarch64 + # Conditional steps below workaroud this by using deadsnakes for that case only + - name: "Install Python for ARM on Linux" + if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }} + uses: deadsnakes/action@v3.0.1 + with: + python-version: '3.8' + - name: "Install Python cases other than ARM on Linux" + if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }} + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: "Print Python version" + run: python3 --version + - name: "Install shiv" + run: pip install shiv + - name: "Generate build version file" + run: ./scripts/create_build_tag_file.sh + - name: "Build local shiv package" + run: ./scripts/build_shiv_package.sh + - name: Start dockerd # Also needed until we can incorporate into the executor + run: | + dockerd -H $DOCKER_HOST --userland-proxy=false & + sleep 5 + - name: "Run fixturenet-laconicd tests" + run: ./tests/fixturenet-laconicd/run-test.sh diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml index fb45c27e..ffbb9937 100644 --- a/.gitea/workflows/publish.yml +++ b/.gitea/workflows/publish.yml @@ -5,6 +5,8 @@ on: branches: - main - publish-test + paths-ignore: + - '.gitea/workflows/triggers/*' jobs: publish: @@ -18,7 +20,15 @@ jobs: run: | build_tag=$(./scripts/create_build_tag_file.sh) echo "build-tag=v${build_tag}" >> $GITHUB_OUTPUT - - name: "Install Python" + # At present the stock setup-python action fails on Linux/aarch64 + # Conditional steps below workaroud this by using deadsnakes for that case only + - name: "Install Python for ARM on Linux" + if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }} + uses: deadsnakes/action@v3.0.1 + with: + python-version: '3.8' + - name: "Install Python cases other than ARM on Linux" + if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }} uses: actions/setup-python@v4 with: python-version: '3.8' diff --git a/.gitea/workflows/test-deploy.yml b/.gitea/workflows/test-deploy.yml index 5ac5b450..6e47c82d 100644 --- a/.gitea/workflows/test-deploy.yml +++ b/.gitea/workflows/test-deploy.yml @@ -7,6 +7,8 @@ on: branches: - main - ci-test + paths-ignore: + - '.gitea/workflows/triggers/*' # Needed until we can incorporate docker startup into the executor container env: @@ -19,7 +21,15 @@ jobs: steps: - name: "Clone project repository" uses: actions/checkout@v3 - - name: "Install Python" + # At present the stock setup-python action fails on Linux/aarch64 + # Conditional steps below workaroud this by using deadsnakes for that case only + - name: "Install Python for ARM on Linux" + if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }} + uses: deadsnakes/action@v3.0.1 + with: + python-version: '3.8' + - name: "Install Python cases other than ARM on Linux" + if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }} uses: actions/setup-python@v4 with: python-version: '3.8' diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml new file mode 100644 index 00000000..84cce91a --- /dev/null +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -0,0 +1,55 @@ +name: K8s Deploy Test + +on: + pull_request: + branches: '*' + push: + branches: + - main + - ci-test + paths-ignore: + - '.gitea/workflows/triggers/*' + +# Needed until we can incorporate docker startup into the executor container +env: + DOCKER_HOST: unix:///var/run/dind.sock + +jobs: + test: + name: "Run deploy test suite" + runs-on: ubuntu-latest + steps: + - name: "Clone project repository" + uses: actions/checkout@v3 + # At present the stock setup-python action fails on Linux/aarch64 + # Conditional steps below workaroud this by using deadsnakes for that case only + - name: "Install Python for ARM on Linux" + if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }} + uses: deadsnakes/action@v3.0.1 + with: + python-version: '3.8' + - name: "Install Python cases other than ARM on Linux" + if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }} + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: "Print Python version" + run: python3 --version + - name: "Install shiv" + run: pip install shiv + - name: "Generate build version file" + run: ./scripts/create_build_tag_file.sh + - name: "Build local shiv package" + run: ./scripts/build_shiv_package.sh + - name: Start dockerd # Also needed until we can incorporate into the executor + run: | + dockerd -H $DOCKER_HOST --userland-proxy=false & + sleep 5 + - name: "Install Go" + uses: actions/setup-go@v4 + with: + go-version: '1.21' + - name: "Install Kind" + run: go install sigs.k8s.io/kind@v0.20.0 + - name: "Debug Kind" + run: kind create cluster --retain && docker logs kind-control-plane diff --git a/.gitea/workflows/test-webapp.yml b/.gitea/workflows/test-webapp.yml new file mode 100644 index 00000000..9fbf84b2 --- /dev/null +++ b/.gitea/workflows/test-webapp.yml @@ -0,0 +1,49 @@ +name: Webapp Test + +on: + pull_request: + branches: '*' + push: + branches: + - main + - ci-test + paths-ignore: + - '.gitea/workflows/triggers/*' + +# Needed until we can incorporate docker startup into the executor container +env: + DOCKER_HOST: unix:///var/run/dind.sock + +jobs: + test: + name: "Run webapp test suite" + runs-on: ubuntu-latest + steps: + - name: "Clone project repository" + uses: actions/checkout@v3 + # At present the stock setup-python action fails on Linux/aarch64 + # Conditional steps below workaroud this by using deadsnakes for that case only + - name: "Install Python for ARM on Linux" + if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }} + uses: deadsnakes/action@v3.0.1 + with: + python-version: '3.8' + - name: "Install Python cases other than ARM on Linux" + if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }} + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: "Print Python version" + run: python3 --version + - name: "Install shiv" + run: pip install shiv + - name: "Generate build version file" + run: ./scripts/create_build_tag_file.sh + - name: "Build local shiv package" + run: ./scripts/build_shiv_package.sh + - name: Start dockerd # Also needed until we can incorporate into the executor + run: | + dockerd -H $DOCKER_HOST --userland-proxy=false & + sleep 5 + - name: "Run webapp tests" + run: ./tests/webapp-test/run-webapp-test.sh diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 46c5f937..193e27c9 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -7,6 +7,8 @@ on: branches: - main - ci-test + paths-ignore: + - '.gitea/workflows/triggers/*' # Needed until we can incorporate docker startup into the executor container env: @@ -19,7 +21,15 @@ jobs: steps: - name: "Clone project repository" uses: actions/checkout@v3 - - name: "Install Python" + # At present the stock setup-python action fails on Linux/aarch64 + # Conditional steps below workaroud this by using deadsnakes for that case only + - name: "Install Python for ARM on Linux" + if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }} + uses: deadsnakes/action@v3.0.1 + with: + python-version: '3.8' + - name: "Install Python cases other than ARM on Linux" + if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }} uses: actions/setup-python@v4 with: python-version: '3.8' @@ -38,3 +48,4 @@ jobs: - name: "Run smoke tests" run: ./tests/smoke-test/run-smoke-test.sh + diff --git a/.gitea/workflows/triggers/fixturenet-eth-plugeth-test b/.gitea/workflows/triggers/fixturenet-eth-plugeth-test new file mode 100644 index 00000000..e0c6da50 --- /dev/null +++ b/.gitea/workflows/triggers/fixturenet-eth-plugeth-test @@ -0,0 +1,2 @@ +Change this file to trigger running the fixturenet-eth-plugeth-test CI job +trigger diff --git a/.gitea/workflows/triggers/fixturenet-eth-test b/.gitea/workflows/triggers/fixturenet-eth-test new file mode 100644 index 00000000..7d12834e --- /dev/null +++ b/.gitea/workflows/triggers/fixturenet-eth-test @@ -0,0 +1,2 @@ +Change this file to trigger running the fixturenet-eth-test CI job + diff --git a/.gitea/workflows/triggers/fixturenet-laconicd-test b/.gitea/workflows/triggers/fixturenet-laconicd-test new file mode 100644 index 00000000..e6b73875 --- /dev/null +++ b/.gitea/workflows/triggers/fixturenet-laconicd-test @@ -0,0 +1,2 @@ +Change this file to trigger running the fixturenet-laconicd-test CI job + diff --git a/.github/workflows/fixturenet-eth.yml b/.github/workflows/fixturenet-eth.yml new file mode 100644 index 00000000..a7f744ac --- /dev/null +++ b/.github/workflows/fixturenet-eth.yml @@ -0,0 +1,30 @@ +name: Fixturenet-Eth Test + +on: + push: + branches: '*' + paths: + - '!**' + - '.github/workflows/triggers/fixturenet-eth-test' + +jobs: + test: + name: "Run fixturenet-eth test suite" + runs-on: ubuntu-latest + steps: + - name: "Clone project repository" + uses: actions/checkout@v3 + - name: "Install Python" + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: "Print Python version" + run: python3 --version + - name: "Install shiv" + run: pip install shiv + - name: "Generate build version file" + run: ./scripts/create_build_tag_file.sh + - name: "Build local shiv package" + run: ./scripts/build_shiv_package.sh + - name: "Run fixturenet-eth tests" + run: ./tests/fixturenet-eth/run-test.sh diff --git a/.github/workflows/fixturenet-laconicd.yml b/.github/workflows/fixturenet-laconicd.yml new file mode 100644 index 00000000..a16c1fe6 --- /dev/null +++ b/.github/workflows/fixturenet-laconicd.yml @@ -0,0 +1,30 @@ +name: Fixturenet-Laconicd Test + +on: + push: + branches: '*' + paths: + - '!**' + - '.github/workflows/triggers/fixturenet-laconicd-test' + +jobs: + test: + name: "Run fixturenet-laconicd test suite" + runs-on: ubuntu-latest + steps: + - name: "Clone project repository" + uses: actions/checkout@v3 + - name: "Install Python" + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: "Print Python version" + run: python3 --version + - name: "Install shiv" + run: pip install shiv + - name: "Generate build version file" + run: ./scripts/create_build_tag_file.sh + - name: "Build local shiv package" + run: ./scripts/build_shiv_package.sh + - name: "Run fixturenet-laconicd tests" + run: ./tests/fixturenet-laconicd/run-test.sh diff --git a/.github/workflows/test-webapp.yml b/.github/workflows/test-webapp.yml new file mode 100644 index 00000000..3b920828 --- /dev/null +++ b/.github/workflows/test-webapp.yml @@ -0,0 +1,29 @@ +name: Webapp Test + +on: + pull_request: + branches: '*' + push: + branches: '*' + +jobs: + test: + name: "Run webapp test suite" + runs-on: ubuntu-latest + steps: + - name: "Clone project repository" + uses: actions/checkout@v3 + - name: "Install Python" + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: "Print Python version" + run: python3 --version + - name: "Install shiv" + run: pip install shiv + - name: "Generate build version file" + run: ./scripts/create_build_tag_file.sh + - name: "Build local shiv package" + run: ./scripts/build_shiv_package.sh + - name: "Run webapp tests" + run: ./tests/webapp-test/run-webapp-test.sh diff --git a/.github/workflows/triggers/fixturenet-eth-test b/.github/workflows/triggers/fixturenet-eth-test new file mode 100644 index 00000000..7d12834e --- /dev/null +++ b/.github/workflows/triggers/fixturenet-eth-test @@ -0,0 +1,2 @@ +Change this file to trigger running the fixturenet-eth-test CI job + diff --git a/.github/workflows/triggers/fixturenet-laconicd-test b/.github/workflows/triggers/fixturenet-laconicd-test new file mode 100644 index 00000000..ad4c76a7 --- /dev/null +++ b/.github/workflows/triggers/fixturenet-laconicd-test @@ -0,0 +1,3 @@ +Change this file to trigger running the fixturenet-laconicd-test CI job + +trigger diff --git a/.gitignore b/.gitignore index 0e253ae4..3aaa220b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,5 @@ laconic_stack_orchestrator.egg-info __pycache__ *~ package -app/data/build_tag.txt -build +stack_orchestrator/data/build_tag.txt +/build diff --git a/README.md b/README.md index ac1beaa2..aa979e3a 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Version: 1.1.0-7a607c2-202304260513 Save the distribution url to `~/.laconic-so/config.yml`: ```bash mkdir ~/.laconic-so -echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" > ~/.laconic-so/config.yml" +echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" > ~/.laconic-so/config.yml ``` ### Update @@ -64,12 +64,12 @@ laconic-so update ## Usage -The various [stacks](/app/data/stacks) each contain instructions for running different stacks based on your use case. For example: +The various [stacks](/stack_orchestrator/data/stacks) each contain instructions for running different stacks based on your use case. For example: -- [self-hosted Gitea](/app/data/stacks/build-support) -- [an Optimism Fixturenet](/app/data/stacks/fixturenet-optimism) -- [laconicd with console and CLI](app/data/stacks/fixturenet-laconic-loaded) -- [kubo (IPFS)](app/data/stacks/kubo) +- [self-hosted Gitea](/stack_orchestrator/data/stacks/build-support) +- [an Optimism Fixturenet](/stack_orchestrator/data/stacks/fixturenet-optimism) +- [laconicd with console and CLI](stack_orchestrator/data/stacks/fixturenet-laconic-loaded) +- [kubo (IPFS)](stack_orchestrator/data/stacks/kubo) ## Contributing diff --git a/app/data/compose/docker-compose-nitro-reverse-payment-proxy.yml b/app/data/compose/docker-compose-nitro-reverse-payment-proxy.yml deleted file mode 100644 index bf7461a0..00000000 --- a/app/data/compose/docker-compose-nitro-reverse-payment-proxy.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: '3.7' - -services: - nitro-reverse-payment-proxy: - image: cerc/go-nitro:local - hostname: nitro-reverse-payment-proxy - restart: on-failure - depends_on: - # Wait for the go-nitro node to start - go-nitro: - condition: service_healthy - environment: - CERC_PROXY_ADDRESS: 0.0.0.0:8081 - CERC_PROXY_NITRO_ENDPOINT: ${CERC_PROXY_NITRO_ENDPOINT:-go-nitro:4005/api/v1} - CERC_PROXY_DESTINATION_URL: ${CERC_PROXY_DESTINATION_URL:-http://ipld-eth-server:8081} - CERC_PROXY_COST_PER_BYTE: ${CERC_PROXY_COST_PER_BYTE:-1} - entrypoint: ["bash", "-c", "/app/run-reverse-payment-proxy.sh"] - volumes: - - ../config/go-nitro/run-reverse-payment-proxy.sh:/app/run-reverse-payment-proxy.sh - healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "8081"] - interval: 10s - timeout: 5s - retries: 10 - start_period: 10s - ports: - - "8081:8081" diff --git a/app/data/compose/docker-compose-ponder.yml b/app/data/compose/docker-compose-ponder.yml deleted file mode 100644 index ca054a7d..00000000 --- a/app/data/compose/docker-compose-ponder.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: '3.7' - -services: - ponder-app-indexer: - hostname: ponder-app-indexer - restart: unless-stopped - image: cerc/ponder:local - working_dir: /app/examples/token-erc20 - environment: - CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99} - CERC_PONDER_RPC_URL_1: ${PONDER_RPC_URL_1:-http://nitro-reverse-payment-proxy:8081} - CERC_PONDER_NITRO_PK: ${CERC_PONDER_INDEXER_NITRO_PK:-58368d20ff12f17669c06158c21d885897aa56f9be430edc789614bf9851d53f} - CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_INDEXER_NITRO_CHAIN_PK:-fb1e9af328c283ca3e2486e7c24d13582b7912057d8b9542ff41503c85bc05c0} - CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546} - CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR} - CERC_UPSTREAM_NITRO_ADDRESS: ${CERC_UPSTREAM_NITRO_ADDRESS:-0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE} - CERC_UPSTREAM_NITRO_MULTIADDR: ${CERC_UPSTREAM_NITRO_MULTIADDR:-/dns4/go-nitro/tcp/5005/ws/p2p/16Uiu2HAmSjXJqsyBJgcBUU2HQmykxGseafSatbpq5471XmuaUqyv} - CERC_UPSTREAM_NITRO_PAY_AMOUNT: ${CERC_UPSTREAM_NITRO_PAY_AMOUNT:-5000} - command: ["bash", "./ponder-start.sh"] - volumes: - - ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh - - ../config/ponder/ponder.indexer.config.ts:/app/examples/token-erc20/ponder.config.ts - - peers_ids:/peers - - nitro_deployment:/nitro - - ponder_indexer_nitro_data:/app/examples/token-erc20/.ponder/nitro-db - ports: - - "42070" - extra_hosts: - - "host.docker.internal:host-gateway" - - ponder-app-watcher: - hostname: ponder-app-watcher - depends_on: - - ponder-app-indexer - restart: unless-stopped - image: cerc/ponder:local - working_dir: /app/examples/token-erc20 - environment: - CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99} - CERC_PONDER_NITRO_PK: ${CERC_PONDER_WATCHER_NITRO_PK:-febb3b74b0b52d0976f6571d555f4ac8b91c308dfa25c7b58d1e6a7c3f50c781} - CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_WATCHER_NITRO_CHAIN_PK:-be4aa664815ea3bc3d63118649a733f6c96b243744310806ecb6d96359ab62cf} - CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546} - CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR} - CERC_INDEXER_GQL_ENDPOINT: ${CERC_INDEXER_GQL_ENDPOINT:-http://ponder-app-indexer:42070/graphql} - CERC_INDEXER_NITRO_ADDRESS: ${CERC_INDEXER_NITRO_ADDRESS:-0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d} - CERC_INDEXER_NITRO_PAY_AMOUNT: ${CERC_INDEXER_NITRO_PAY_AMOUNT:-50} - command: ["bash", "./ponder-start.sh"] - volumes: - - ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh - - ../config/ponder/ponder.watcher.config.ts:/app/examples/token-erc20/ponder.config.ts - - peers_ids:/peers - - nitro_deployment:/nitro - - ponder_watcher_nitro_data:/app/examples/token-erc20/.ponder/nitro-db - ports: - - "42069" - extra_hosts: - - "host.docker.internal:host-gateway" - -volumes: - peers_ids: - nitro_deployment: - ponder_indexer_nitro_data: - ponder_watcher_nitro_data: diff --git a/app/data/config/go-nitro/run-reverse-payment-proxy.sh b/app/data/config/go-nitro/run-reverse-payment-proxy.sh deleted file mode 100755 index 4aa07fbe..00000000 --- a/app/data/config/go-nitro/run-reverse-payment-proxy.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -e -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi - -echo "Running Nitro reverse payment proxy" -echo "Using CERC_PROXY_ADDRESS ${CERC_PROXY_ADDRESS}" -echo "Using CERC_PROXY_NITRO_ENDPOINT ${CERC_PROXY_NITRO_ENDPOINT}" -echo "Using CERC_PROXY_DESTINATION_URL ${CERC_PROXY_DESTINATION_URL}" -echo "Using CERC_PROXY_COST_PER_BYTE ${CERC_PROXY_COST_PER_BYTE}" - -./proxy -proxyaddress ${CERC_PROXY_ADDRESS} -nitroendpoint=${CERC_PROXY_NITRO_ENDPOINT} -destinationurl=${CERC_PROXY_DESTINATION_URL} -costperbyte ${CERC_PROXY_COST_PER_BYTE} -enablepaidrpcmethods diff --git a/app/data/container-build/cerc-go-nitro/build.sh b/app/data/container-build/cerc-go-nitro/build.sh deleted file mode 100755 index 0637f46d..00000000 --- a/app/data/container-build/cerc-go-nitro/build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -# Build cerc/go-nitro - -source ${CERC_CONTAINER_BASE_DIR}/build-base.sh - -docker build -t cerc/go-nitro:local -f ${CERC_REPO_BASE_DIR}/go-nitro/docker/local/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/go-nitro diff --git a/app/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh b/app/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh deleted file mode 100755 index 0d90da8d..00000000 --- a/app/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Create some demo/test records in the registry -set -e -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi -registry_command="laconic cns" -record_1_filename=demo-record-1.yml -cat < ${record_1_filename} -record: - type: WebsiteRegistrationRecord - url: 'https://cerc.io' - repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D - build_artifact_cid: QmP8jTG1m9GSDJLCbeWhVSVgEzCPPwXRdCRuJtQ5Tz9Kc9 - tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR - version: 1.0.23 -EOF -# Check we have funds -funds_response=$(${registry_command} account get --address $(cat my-address.txt)) -funds_balance=$(echo ${funds_response} | jq -r .[0].balance[0].quantity) -echo "Balance is: ${funds_balance}" -# Create a bond -bond_create_result=$(${registry_command} bond create --type aphoton --quantity 1000000000) -bond_id=$(echo ${bond_create_result} | jq -r .bondId) -echo "Created bond with id: ${bond_id}" -# Publish a demo record -publish_response=$(${registry_command} record publish --filename ${record_1_filename} --bond-id ${bond_id}) -published_record_id=$(echo ${publish_response} | jq -r .id) -echo "Published ${record_1_filename} with id: ${published_record_id}" diff --git a/app/data/container-build/cerc-watcher-erc20/Dockerfile b/app/data/container-build/cerc-watcher-erc20/Dockerfile deleted file mode 100644 index 5ec94987..00000000 --- a/app/data/container-build/cerc-watcher-erc20/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM node:16.17.1-alpine3.16 - -RUN apk --update --no-cache add git python3 alpine-sdk - -WORKDIR /app - -COPY . . - -RUN echo "Building watcher-ts" && \ - git checkout v0.2.19 && \ - yarn && yarn build - -WORKDIR /app/packages/erc20-watcher diff --git a/app/data/stacks/fixturenet-payments/README.md b/app/data/stacks/fixturenet-payments/README.md deleted file mode 100644 index cb2e9422..00000000 --- a/app/data/stacks/fixturenet-payments/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# fixturenet-payments - -Instructions to setup and deploy an end-to-end fixturenet-payments stack - -## Setup - -Clone required repositories: - -```bash -laconic-so --stack fixturenet-payments setup-repositories --pull -``` - -Build the container images: - -```bash -laconic-so --stack fixturenet-payments build-containers -``` - -## Deploy - -Deploy the stack: - -```bash -laconic-so --stack fixturenet-payments deploy --cluster payments up - -# Exposed on host ports: -# 4005: go-nitro node's RPC endpoint -# 5005: go-nitro node's p2p endpoint -# 8081: reverse payment proxy's RPC endpoint -# 15432: MobyMask v3 watcher's db endpoint -# 3001: MobyMask v3 watcher endpoint -# 9090: MobyMask v3 watcher relay node endpoint -# 8080: MobyMask snap -# 3004: MobyMask v3 app -``` - -## Demo - -Follow the [demo](./demo.md) to try out end-to-end payments - -## Clean up - -Stop all the services running in background: - -```bash -laconic-so --stack fixturenet-payments deploy --cluster payments down 30 -``` - -Clear volumes created by this stack: - -```bash -# List all relevant volumes -docker volume ls -q --filter "name=[payments" - -# Remove all the listed volumes -docker volume rm $(docker volume ls -q --filter "name=[payments") -``` diff --git a/app/data/stacks/fixturenet-payments/demo.md b/app/data/stacks/fixturenet-payments/demo.md deleted file mode 100644 index fc01f3b1..00000000 --- a/app/data/stacks/fixturenet-payments/demo.md +++ /dev/null @@ -1,310 +0,0 @@ -# Demo - -Stack components: -* `ipld-eth-db` database for statediffed data -* Local geth + lighthouse blockchain "fixturenet" running in statediffing mode -* `ipld-eth-server` which runs an ETH RPC API and a GQL server; serves data from `ipld-eth-db` -* A go-nitro deployment acting as the Nitro node for `ipld-eth-server` -* A modified reverse payment proxy server (based on the one from go-nitro) that proxies requests to `ipld-eth-server`'s RPC endpoint; it talks to `ipld-eth-server`'s Nitro node to accept and validate payments required for configured RPC requests -* A MobyMask v3 watcher that pays the `ipld-eth-server` for ETH RPC requests -* A MobyMask v3 app that pays the watcher for reads (GQL queries) and writes -* An example ERC20 Ponder app that pays the `ipld-eth-server` for ETH RPC requests - -## Setup - -* On starting the stack, MobyMask watcher creates a payment channel with the `ipld-eth-server`'s Nitro node. Check watcher logs and wait for the same: - - ```bash - docker logs -f $(docker ps -aq --filter name="mobymask-watcher-server") - - # Expected output: - # vulcanize:server Peer ID: 12D3KooWKLqLWU82VU7jmsmQMruRvZWhoBoVsf1UHchM5Nuq9ymY - # vulcanize:server Using chain URL http://fixturenet-eth-geth-1:8546 for Nitro node - # ... - # ts-nitro:util:nitro Ledger channel created with id 0x65703ccdfacab09ac35367bdbe6c5a337e7a6651aad526807607b1c59b28bc1e - # ... - # ts-nitro:util:nitro Virtual payment channel created with id 0x29ff1335d73391a50e8fde3e9b34f00c3d81c39ddc7f89187f44dd51df96140e - # vulcanize:server Starting server... +0ms - ``` - -* Keep the above command running to keep track of incoming payments and GQL requests from the MobyMask app - -* In another terminal, export the payment channel id to a variable: - - ```bash - export WATCHER_UPSTREAM_PAYMENT_CHANNEL= - ``` - -* Check the payment channel status: - - ```bash - docker exec payments-nitro-rpc-client-1 npm exec -c "nitro-rpc-client get-payment-channel $WATCHER_UPSTREAM_PAYMENT_CHANNEL -h go-nitro -p 4005" - - # Expected output: - # { - # ID: '0x8c0d17639bd2ba07dbcd248304a8f3c6c7276bfe25c2b87fe41f461e20f33f01', - # Status: 'Open', - # Balance: { - # AssetAddress: '0x0000000000000000000000000000000000000000', - # Payee: '0xaaa6628ec44a8a742987ef3a114ddfe2d4f7adce', - # Payer: '0xbbb676f9cff8d242e9eac39d063848807d3d1d94', - # PaidSoFar: 0n, - # RemainingFunds: 1000000000n - # } - # } - ``` - -* In another terminal, check the reverse payment proxy server's logs to keep track of incoming payments and RPC requests: - - ```bash - docker logs -f $(docker ps -aq --filter name="nitro-reverse-payment-proxy") - ``` - -* MetaMask flask wallet setup for running the MobyMask app: - - * Get the geth node’s port mapped to host: - - ```bash - docker port payments-fixturenet-eth-geth-1-1 8545 - ``` - - * In MetaMask, add a custom network with the following settings: - - ```bash - # Network name - Local fixturenet - - # New RPC URL - http://127.0.0.1: - - # Chain ID - 1212 - - # Currency symbol - ETH - ``` - - * Import a faucet account with the following private key: - - ```bash - # Faucet PK - # 0x570b909da9669b2f35a0b1ac70b8358516d55ae1b5b3710e95e9a94395090597 - ``` - - * Create an additional account for usage in the app; fund it from the faucet account - -* Get the generated root invite link for MobyMask from contract deployment container logs: - - ```bash - docker logs -f $(docker ps -aq --filter name="mobymask-1") - - # Expected output: - # ... - # "key": "0x60e706fda4639fe0a8eb102cb0ce81231cf6e819f41cb4eadf72d865ea4c11ad" - # } - # http://127.0.0.1:3004/#/members?invitation= - ``` - -## Run - -### MobyMask App - -* Open app in a browser (where MetaMask was setup) using the invite link - -* Run the following in browser console to enable logs: - - ```bash - localStorage.debug = 'ts-nitro:*' - # Refresh the tab for taking effect - ``` - -* In the app’s debug panel, check that the peer gets connected to relay node and watcher peer - -* Open the `NITRO` tab in debug panel - * Click on `Connect Wallet` to connect to MetaMask (make sure that the newly funded account is active) - * Click on `Connect Snap` to install/connect snap - -* Perform `DIRECT FUND` with the preset amount and wait for the MetaMask confirmation prompt to appear; confirm the transaction and wait for a ledger channel to be created with the watcher - -* Perform `VIRTUAL FUND` with amount set to `10000` and wait for a payment channel to be created with the watcher - -* Perform phisher status check queries now that a payment channel is created: - - * Check the watcher logs for received payments along with the GQL queries: - - ```bash - # Expected output: - # ... - # laconic:payments Serving a paid query for 0x86804299822212c070178B5135Ba6DdAcFC357D3 - # vulcanize:resolver isPhisher 0x98ae4f9e9d01cc892adfe6871e1db0287039e0c183d3b5bb31d724228c114744 0x2B6AFbd4F479cE4101Df722cF4E05F941523EaD9 TWT:ash1 - # vulcanize:indexer isPhisher: db miss, fetching from upstream server - # laconic:payments Making RPC call: eth_chainId - # laconic:payments Making RPC call: eth_getBlockByHash - # laconic:payments Making RPC call: eth_chainId - # laconic:payments Making RPC call: eth_getStorageAt - ``` - - * The watcher makes several ETH RPC requests to `ipld-eth-server` to fetch data required for satisfying the GQL request(s); check the payment proxy server logs for charged RPC requests (`eth_getBlockByHash`, `eth_getBlockByNumber`, `eth_getStorageAt`): - - ```bash - # Expected output: - # ... - # {"time":"2023-10-06T06:46:52.769009314Z","level":"DEBUG","msg":"Serving RPC request","method":"eth_chainId"} - # {"time":"2023-10-06T06:46:52.773006426Z","level":"DEBUG","msg":"Serving RPC request","method":"eth_getBlockByNumber"} - # {"time":"2023-10-06T06:46:52.811142054Z","level":"DEBUG","msg":"Request cost","cost-per-byte":1,"response-length":1480,"cost":1480,"method":"eth_getBlockByNumber"} - # {"time":"2023-10-06T06:46:52.811418494Z","level":"DEBUG","msg":"sent message","address":"0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE","method":"receive_voucher"} - # {"time":"2023-10-06T06:46:52.812557482Z","level":"DEBUG","msg":"Received voucher","delta":5000} - # ... - # {"time":"2023-10-06T06:46:52.87525215Z","level":"DEBUG","msg":"Serving RPC request","method":"eth_getStorageAt"} - # {"time":"2023-10-06T06:46:52.882859654Z","level":"DEBUG","msg":"Request cost","cost-per-byte":1,"response-length":104,"cost":104,"method":"eth_getStorageAt"} - # {"time":"2023-10-06T06:46:52.882946485Z","level":"DEBUG","msg":"sent message","address":"0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE","method":"receive_voucher"} - # {"time":"2023-10-06T06:46:52.884012641Z","level":"DEBUG","msg":"Received voucher","delta":5000} - # {"time":"2023-10-06T06:46:52.884032961Z","level":"DEBUG","msg":"Destination request","url":"http://ipld-eth-server:8081/"} - ``` - -* Change the amount besides `PAY` button in debug panel to `>=100` for phisher reports next - -* Perform a phisher report and check the watcher logs for received payments: - - ```bash - # Expected output: - # ... - # vulcanize:libp2p-utils [6:50:2] Received a message on mobymask P2P network from peer: 12D3KooWRkxV9SX8uTUZYkbRjai4Fsn7yavB61J5TMnksixsabsP - # ts-nitro:engine {"msg":"Received message","_msg":{"to":"0xBBB676","from":"0x868042","payloadSummaries":[],"proposalSummaries":[],"payments":[{"amount":200,"channelId":"0x557153d729cf3323c0bdb40a36b245f98c2d4562933ba2182c9d61c5cfeda948"}],"rejectedObjectives":[]}} - # laconic:payments Received a payment voucher of 100 from 0x86804299822212c070178B5135Ba6DdAcFC357D3 - # vulcanize:libp2p-utils Payment received for a mutation request from 0x86804299822212c070178B5135Ba6DdAcFC357D3 - # vulcanize:libp2p-utils Transaction receipt for invoke message { - # to: '0x2B6AFbd4F479cE4101Df722cF4E05F941523EaD9', - # blockNumber: 232, - # blockHash: '0x6a188722c102662ea48af3786fe9db0d4b6c7ab7b27473eb0e628cf95746a244', - # transactionHash: '0x6521205db8a905b3222adc2b6855f9b2abc72580624d299bec2a35bcba173efa', - # effectiveGasPrice: '1500000007', - # gasUsed: '113355' - # } - ``` - -* Check the watcher - ipld-eth-server payment channel status after a few requests: - - ```bash - docker exec payments-nitro-rpc-client-1 npm exec -c "nitro-rpc-client get-payment-channel $WATCHER_UPSTREAM_PAYMENT_CHANNEL -h go-nitro -p 4005" - - # Expected output ('PaidSoFar' should be non zero): - # { - # ID: '0x8c0d17639bd2ba07dbcd248304a8f3c6c7276bfe25c2b87fe41f461e20f33f01', - # Status: 'Open', - # Balance: { - # AssetAddress: '0x0000000000000000000000000000000000000000', - # Payee: '0xaaa6628ec44a8a742987ef3a114ddfe2d4f7adce', - # Payer: '0xbbb676f9cff8d242e9eac39d063848807d3d1d94', - # PaidSoFar: 30000n, - # RemainingFunds: 999970000n - # } - # } - ``` - -### ERC20 Ponder App - -* Run the ponder app in indexer mode: - - ```bash - docker exec -it payments-ponder-app-indexer-1 bash -c "DEBUG=laconic:payments pnpm start" - - # Expected output: - # 08:00:28.701 INFO payment Nitro node setup with address 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d - # laconic:payments Starting voucher subscription... +0ms - # ... - # 09:58:54.288 INFO payment Creating ledger channel with nitro node 0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE - # ... - # 09:59:14.230 INFO payment Creating payment channel with nitro node 0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE - # ... - # 09:59:14.329 INFO payment Using payment channel 0x10f049519bc3f862e2b26e974be8666886228f30ea54aab06e2f23718afffab0 - ``` - -* Export the payment channel id to a variable: - - ```bash - export PONDER_UPSTREAM_PAYMENT_CHANNEL= - ``` - -* On starting the Ponder app in indexer mode, it creates a payment channel with the `ipld-eth-server`'s Nitro node and then starts the historical sync service - -* The sync service makes several ETH RPC requests to the `ipld-eth-server` to fetch required data; check the payment proxy server logs for charged RPC requests (`eth_getBlockByNumber`, `eth_getLogs`) - - ```bash - # Expected output: - # ... - # {"time":"2023-10-06T06:51:45.214478402Z","level":"DEBUG","msg":"Serving RPC request","method":"eth_getBlockByNumber"} - # {"time":"2023-10-06T06:51:45.22251171Z","level":"DEBUG","msg":"Request cost","cost-per-byte":1,"response-length":576,"cost":576,"method":"eth_getBlockByNumber"} - # {"time":"2023-10-06T06:51:45.222641963Z","level":"DEBUG","msg":"sent message","address":"0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE","method":"receive_voucher"} - # {"time":"2023-10-06T06:51:45.224042391Z","level":"DEBUG","msg":"Received voucher","delta":5000} - # {"time":"2023-10-06T06:51:45.224061411Z","level":"DEBUG","msg":"Destination request","url":"http://ipld-eth-server:8081/"} - # {"time":"2023-10-06T06:51:45.242064953Z","level":"DEBUG","msg":"Serving RPC request","method":"eth_getLogs"} - # {"time":"2023-10-06T06:51:45.249118517Z","level":"DEBUG","msg":"Request cost","cost-per-byte":1,"response-length":61,"cost":61,"method":"eth_getLogs"} - # {"time":"2023-10-06T06:51:45.249189892Z","level":"DEBUG","msg":"sent message","address":"0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE","method":"receive_voucher"} - # {"time":"2023-10-06T06:51:45.249743149Z","level":"DEBUG","msg":"Received voucher","delta":5000} - # {"time":"2023-10-06T06:51:45.249760631Z","level":"DEBUG","msg":"Destination request","url":"http://ipld-eth-server:8081/"} - # ... - ``` - -* Check the ponder - ipld-eth-server payment channel status: - - ```bash - docker exec payments-nitro-rpc-client-1 npm exec -c "nitro-rpc-client get-payment-channel $PONDER_UPSTREAM_PAYMENT_CHANNEL -h go-nitro -p 4005" - - # Expected output ('PaidSoFar' is non zero): - # { - # ID: '0x1178ac0f2a43e54a122216fa6afdd30333b590e49e50317a1f9274a591da0f96', - # Status: 'Open', - # Balance: { - # AssetAddress: '0x0000000000000000000000000000000000000000', - # Payee: '0xaaa6628ec44a8a742987ef3a114ddfe2d4f7adce', - # Payer: '0x67d5b55604d1af90074fcb69b8c51838fff84f8d', - # PaidSoFar: 215000n, - # RemainingFunds: 999785000n - # } - # } - ``` - -* In another terminal run the ponder app in watcher mode: - ```bash - docker exec -it payments-ponder-app-watcher-1 bash -c "DEBUG=laconic:payments pnpm start" - - # Expected output: - # 11:23:22.057 DEBUG app Started using config file: ponder.config.ts - # 08:02:12.548 INFO payment Nitro node setup with address 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db - # laconic:payments Starting voucher subscription... +0ms - # 08:02:17.417 INFO payment Creating ledger channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ... - # 08:02:37.135 INFO payment Creating payment channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ... - # 08:02:37.313 INFO payment Using payment channel 0x4b8e67f6a6fcfe114fdd60b85f963344ece4c77d4eea3825688c74b45ff5509b - # ... - # 11:23:22.436 INFO server Started responding as healthy - ``` - -* Check the terminal in which indexer mode ponder is running. Logs of payment for `eth_getLogs` queries can be seen: - ```bash - # ... - # 08:02:37.763 DEBUG realtime Finished processing new head block 89 (network=fixturenet) - # laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +444ms - # laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +1ms - # 08:02:37.804 DEBUG payment Verified payment for GQL queries getLogEvents - # laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +45ms - # laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +0ms - # 08:02:37.849 DEBUG payment Verified payment for GQL queries getLogEvents - ``` - -## Clean Up - -* In the MobyMask app, perform `VIRTUAL DEFUND` and `DIRECT DEFUND` (in order) for closing the payment channel created with watcher - -* Run the following in the browser console to delete the Nitro node's data: - - ```bash - await clearNodeStorage() - ``` - -* Run the following in the browser console to clear data in local storage: - ```bash - localStorage.clear() - ``` - -* On a fresh restart, clear activity tab data in MetaMask for concerned accounts diff --git a/docs/adding-a-new-stack.md b/docs/adding-a-new-stack.md index 4fbf27b2..2b2d1a65 100644 --- a/docs/adding-a-new-stack.md +++ b/docs/adding-a-new-stack.md @@ -8,7 +8,7 @@ Core to the feature completeness of stack orchestrator is to [decouple the tool ## Example -- in `app/data/stacks/my-new-stack/stack.yml` add: +- in `stack_orchestrator/data/stacks/my-new-stack/stack.yml` add: ```yaml version: "0.1" @@ -21,7 +21,7 @@ pods: - my-new-stack ``` -- in `app/data/container-build/cerc-my-new-stack/build.sh` add: +- in `stack_orchestrator/data/container-build/cerc-my-new-stack/build.sh` add: ```yaml #!/usr/bin/env bash @@ -30,7 +30,7 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh docker build -t cerc/my-new-stack:local -f ${CERC_REPO_BASE_DIR}/my-new-stack/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/my-new-stack ``` -- in `app/data/compose/docker-compose-my-new-stack.yml` add: +- in `stack_orchestrator/data/compose/docker-compose-my-new-stack.yml` add: ```yaml version: "3.2" @@ -43,20 +43,20 @@ services: - "0.0.0.0:3000:3000" ``` -- in `app/data/repository-list.txt` add: +- in `stack_orchestrator/data/repository-list.txt` add: ```bash github.com/my-org/my-new-stack ``` whereby that repository contains your source code and a `Dockerfile`, and matches the `repos:` field in the `stack.yml`. -- in `app/data/container-image-list.txt` add: +- in `stack_orchestrator/data/container-image-list.txt` add: ```bash cerc/my-new-stack ``` -- in `app/data/pod-list.txt` add: +- in `stack_orchestrator/data/pod-list.txt` add: ```bash my-new-stack diff --git a/docs/laconicd-with-console.md b/docs/laconicd-with-console.md index 79a6336a..e669e1b9 100644 --- a/docs/laconicd-with-console.md +++ b/docs/laconicd-with-console.md @@ -2,10 +2,10 @@ The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following 4 repositories had to be cloned and setup manually: -- https://github.com/cerc-io/laconicd -- https://github.com/cerc-io/laconic-sdk -- https://github.com/cerc-io/laconic-registry-cli -- https://github.com/cerc-io/laconic-console +- https://git.vdb.to/cerc-io/laconicd +- https://git.vdb.to/cerc-io/laconic-sdk +- https://git.vdb.to/cerc-io/laconic-registry-cli +- https://git.vdb.to/cerc-io/laconic-console Now, with Stack Orchestrator, it is a few quick commands. Additionally, the `docker` and `docker compose` integration on the back-end allows the stack to easily persist, facilitating workflows. @@ -52,7 +52,7 @@ laconic-so version 1. Get the repositories ``` -laconic-so --stack fixturenet-laconic-loaded setup-repositories --include github.com/cerc-io/laconicd,github.com/cerc-io/laconic-sdk,github.com/cerc-io/laconic-registry-cli,github.com/cerc-io/laconic-console +laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console ``` 2. Set this environment variable to the Laconic self-hosted Gitea instance: @@ -212,7 +212,7 @@ record: 3. Try out additional CLI commands -- these are documented [here](https://github.com/cerc-io/laconic-registry-cli#readme) and updates are forthcoming +- these are documented [here](https://git.vdb.to/cerc-io/laconic-registry-cli#readme) and updates are forthcoming - e.g,: ``` diff --git a/requirements.txt b/requirements.txt index 7f60f9dd..bbf97b4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ python-decouple>=3.8 +python-dotenv==1.0.0 GitPython>=3.1.32 tqdm>=4.65.0 python-on-whales>=0.64.0 @@ -8,3 +9,4 @@ ruamel.yaml>=0.17.32 pydantic==1.10.9 tomli==2.0.1 validators==0.22.0 +kubernetes>=28.1.0 diff --git a/scripts/create_build_tag_file.sh b/scripts/create_build_tag_file.sh index c814a420..077abf31 100755 --- a/scripts/create_build_tag_file.sh +++ b/scripts/create_build_tag_file.sh @@ -1,6 +1,6 @@ -build_tag_file_name=./app/data/build_tag.txt +build_tag_file_name=./stack_orchestrator/data/build_tag.txt echo "# This file should be re-generated running: scripts/create_build_tag_file.sh script" > $build_tag_file_name -product_version_string=$( tail -1 ./app/data/version.txt ) +product_version_string=$( tail -1 ./stack_orchestrator/data/version.txt ) commit_string=$( git rev-parse --short HEAD ) timestamp_string=$(date +'%Y%m%d%H%M') build_tag_string=${product_version_string}-${commit_string}-${timestamp_string} diff --git a/setup.py b/setup.py index 86050fbc..d89dfc4d 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( long_description=long_description, long_description_content_type="text/markdown", url='https://github.com/cerc-io/stack-orchestrator', - py_modules=['cli', 'app'], + py_modules=['stack_orchestrator'], packages=find_packages(), install_requires=[requirements], python_requires='>=3.7', @@ -25,6 +25,6 @@ setup( "Operating System :: OS Independent", ], entry_points={ - 'console_scripts': ['laconic-so=cli:cli'], + 'console_scripts': ['laconic-so=stack_orchestrator.main:cli'], } ) diff --git a/app/__init__.py b/stack_orchestrator/__init__.py similarity index 100% rename from app/__init__.py rename to stack_orchestrator/__init__.py diff --git a/app/__main__.py b/stack_orchestrator/__main__.py similarity index 100% rename from app/__main__.py rename to stack_orchestrator/__main__.py diff --git a/app/base.py b/stack_orchestrator/base.py similarity index 98% rename from app/base.py rename to stack_orchestrator/base.py index cc20da1b..811d085d 100644 --- a/app/base.py +++ b/stack_orchestrator/base.py @@ -15,7 +15,7 @@ import os from abc import ABC, abstractmethod -from app.deploy import get_stack_status +from stack_orchestrator.deploy.deploy import get_stack_status from decouple import config diff --git a/app/data/__init__.py b/stack_orchestrator/build/__init__.py similarity index 100% rename from app/data/__init__.py rename to stack_orchestrator/build/__init__.py diff --git a/app/build_containers.py b/stack_orchestrator/build/build_containers.py similarity index 56% rename from app/build_containers.py rename to stack_orchestrator/build/build_containers.py index 0187bb5e..5b2748cc 100644 --- a/app/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -27,12 +27,79 @@ import subprocess import click import importlib.resources from pathlib import Path -from app.util import include_exclude_check, get_parsed_stack_config -from app.base import get_npm_registry_url +from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config +from stack_orchestrator.base import get_npm_registry_url # TODO: find a place for this # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" +def make_container_build_env(dev_root_path: str, + container_build_dir: str, + debug: bool, + force_rebuild: bool, + extra_build_args: str): + container_build_env = { + "CERC_NPM_REGISTRY_URL": get_npm_registry_url(), + "CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""), + "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""), + "CERC_REPO_BASE_DIR": dev_root_path, + "CERC_CONTAINER_BASE_DIR": container_build_dir, + "CERC_HOST_UID": f"{os.getuid()}", + "CERC_HOST_GID": f"{os.getgid()}", + "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0") + } + container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {}) + container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {}) + container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {}) + docker_host_env = os.getenv("DOCKER_HOST") + if docker_host_env: + container_build_env.update({"DOCKER_HOST": docker_host_env}) + + return container_build_env + + +def process_container(container, + container_build_dir: str, + container_build_env: dict, + dev_root_path: str, + quiet: bool, + verbose: bool, + dry_run: bool, + continue_on_error: bool, + ): + if not quiet: + print(f"Building: {container}") + build_dir = os.path.join(container_build_dir, container.replace("/", "-")) + build_script_filename = os.path.join(build_dir, "build.sh") + if verbose: + print(f"Build script filename: {build_script_filename}") + if os.path.exists(build_script_filename): + build_command = build_script_filename + else: + if verbose: + print(f"No script file found: {build_script_filename}, using default build script") + repo_dir = container.split('/')[1] + # TODO: make this less of a hack -- should be specified in some metadata somewhere + # Check if we have a repo for this container. If not, set the context dir to the container-build subdir + repo_full_path = os.path.join(dev_root_path, repo_dir) + repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir + build_command = os.path.join(container_build_dir, + "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}" + if not dry_run: + if verbose: + print(f"Executing: {build_command} with environment: {container_build_env}") + build_result = subprocess.run(build_command, shell=True, env=container_build_env) + if verbose: + print(f"Return code is: {build_result.returncode}") + if build_result.returncode != 0: + print(f"Error running build for {container}") + if not continue_on_error: + print("FATAL Error: container build failed and --continue-on-error not set, exiting") + sys.exit(1) + else: + print("****** Container Build Error, continuing because --continue-on-error is set") + else: + print("Skipped") @click.command() @click.option('--include', help="only build these containers") @@ -52,7 +119,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): continue_on_error = ctx.obj.continue_on_error # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - container_build_dir = Path(__file__).absolute().parent.joinpath("data", "container-build") + container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build") if local_stack: dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] @@ -67,7 +134,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): print('Dev root directory doesn\'t exist, creating') # See: https://stackoverflow.com/a/20885799/1701505 - from app import data + from stack_orchestrator import data with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: all_containers = container_list_file.read().splitlines() @@ -83,61 +150,16 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): if stack: print(f"Stack: {stack}") - # TODO: make this configurable - container_build_env = { - "CERC_NPM_REGISTRY_URL": get_npm_registry_url(), - "CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""), - "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""), - "CERC_REPO_BASE_DIR": dev_root_path, - "CERC_CONTAINER_BASE_DIR": container_build_dir, - "CERC_HOST_UID": f"{os.getuid()}", - "CERC_HOST_GID": f"{os.getgid()}", - "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0") - } - container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {}) - container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {}) - container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {}) - docker_host_env = os.getenv("DOCKER_HOST") - if docker_host_env: - container_build_env.update({"DOCKER_HOST": docker_host_env}) - - def process_container(container): - if not quiet: - print(f"Building: {container}") - build_dir = os.path.join(container_build_dir, container.replace("/", "-")) - build_script_filename = os.path.join(build_dir, "build.sh") - if verbose: - print(f"Build script filename: {build_script_filename}") - if os.path.exists(build_script_filename): - build_command = build_script_filename - else: - if verbose: - print(f"No script file found: {build_script_filename}, using default build script") - repo_dir = container.split('/')[1] - # TODO: make this less of a hack -- should be specified in some metadata somewhere - # Check if we have a repo for this container. If not, set the context dir to the container-build subdir - repo_full_path = os.path.join(dev_root_path, repo_dir) - repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir - build_command = os.path.join(container_build_dir, "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}" - if not dry_run: - if verbose: - print(f"Executing: {build_command} with environment: {container_build_env}") - build_result = subprocess.run(build_command, shell=True, env=container_build_env) - if verbose: - print(f"Return code is: {build_result.returncode}") - if build_result.returncode != 0: - print(f"Error running build for {container}") - if not continue_on_error: - print("FATAL Error: container build failed and --continue-on-error not set, exiting") - sys.exit(1) - else: - print("****** Container Build Error, continuing because --continue-on-error is set") - else: - print("Skipped") + container_build_env = make_container_build_env(dev_root_path, + container_build_dir, + debug, + force_rebuild, + extra_build_args) for container in containers_in_scope: if include_exclude_check(container, include, exclude): - process_container(container) + process_container(container, container_build_dir, container_build_env, + dev_root_path, quiet, verbose, dry_run, continue_on_error) else: if verbose: print(f"Excluding: {container}") diff --git a/app/build_npms.py b/stack_orchestrator/build/build_npms.py similarity index 97% rename from app/build_npms.py rename to stack_orchestrator/build/build_npms.py index 2ffbea1b..c8e3af43 100644 --- a/app/build_npms.py +++ b/stack_orchestrator/build/build_npms.py @@ -25,8 +25,8 @@ from decouple import config import click import importlib.resources from python_on_whales import docker, DockerException -from app.base import get_stack -from app.util import include_exclude_check, get_parsed_stack_config +from stack_orchestrator.base import get_stack +from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config builder_js_image_name = "cerc/builder-js:local" @@ -83,7 +83,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): os.makedirs(build_root_path) # See: https://stackoverflow.com/a/20885799/1701505 - from app import data + from stack_orchestrator import data with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file: all_packages = package_list_file.read().splitlines() diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py new file mode 100644 index 00000000..f4668c5d --- /dev/null +++ b/stack_orchestrator/build/build_webapp.py @@ -0,0 +1,76 @@ +# Copyright © 2022, 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Builds webapp containers + +# env vars: +# CERC_REPO_BASE_DIR defaults to ~/cerc + +# TODO: display the available list of containers; allow re-build of either all or specific containers + +import os +from decouple import config +import click +from pathlib import Path +from stack_orchestrator.build import build_containers + + +@click.command() +@click.option('--base-container', default="cerc/nextjs-base") +@click.option('--source-repo', help="directory containing the webapp to build", required=True) +@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild") +@click.option("--extra-build-args", help="Supply extra arguments to build") +@click.pass_context +def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): + '''build the specified webapp container''' + + quiet = ctx.obj.quiet + verbose = ctx.obj.verbose + dry_run = ctx.obj.dry_run + debug = ctx.obj.debug + local_stack = ctx.obj.local_stack + stack = ctx.obj.stack + continue_on_error = ctx.obj.continue_on_error + + # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build") + + if local_stack: + dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] + print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + else: + dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + + if not quiet: + print(f'Dev Root is: {dev_root_path}') + + # First build the base container. + container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug, + force_rebuild, extra_build_args) + + build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + verbose, dry_run, continue_on_error) + + + # Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir. + container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo) + container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir, + base_container.replace("/", "-"), + "Dockerfile.webapp") + webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] + container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" + + build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + verbose, dry_run, continue_on_error) diff --git a/app/command_types.py b/stack_orchestrator/command_types.py similarity index 100% rename from app/command_types.py rename to stack_orchestrator/command_types.py diff --git a/app/data/stacks/chain-chunker/README.md b/stack_orchestrator/data/__init__.py similarity index 100% rename from app/data/stacks/chain-chunker/README.md rename to stack_orchestrator/data/__init__.py diff --git a/app/data/compose/docker-compose-contract-sushiswap.yml b/stack_orchestrator/data/compose/docker-compose-contract-sushiswap.yml similarity index 100% rename from app/data/compose/docker-compose-contract-sushiswap.yml rename to stack_orchestrator/data/compose/docker-compose-contract-sushiswap.yml diff --git a/app/data/compose/docker-compose-contract.yml b/stack_orchestrator/data/compose/docker-compose-contract.yml similarity index 100% rename from app/data/compose/docker-compose-contract.yml rename to stack_orchestrator/data/compose/docker-compose-contract.yml diff --git a/app/data/compose/docker-compose-eth-probe.yml b/stack_orchestrator/data/compose/docker-compose-eth-probe.yml similarity index 100% rename from app/data/compose/docker-compose-eth-probe.yml rename to stack_orchestrator/data/compose/docker-compose-eth-probe.yml diff --git a/app/data/compose/docker-compose-eth-statediff-fill-service.yml b/stack_orchestrator/data/compose/docker-compose-eth-statediff-fill-service.yml similarity index 100% rename from app/data/compose/docker-compose-eth-statediff-fill-service.yml rename to stack_orchestrator/data/compose/docker-compose-eth-statediff-fill-service.yml diff --git a/app/data/compose/docker-compose-fixturenet-eth-metrics.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-eth-metrics.yml similarity index 100% rename from app/data/compose/docker-compose-fixturenet-eth-metrics.yml rename to stack_orchestrator/data/compose/docker-compose-fixturenet-eth-metrics.yml diff --git a/app/data/compose/docker-compose-fixturenet-eth.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-eth.yml similarity index 100% rename from app/data/compose/docker-compose-fixturenet-eth.yml rename to stack_orchestrator/data/compose/docker-compose-fixturenet-eth.yml diff --git a/app/data/compose/docker-compose-fixturenet-laconic-console.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml similarity index 100% rename from app/data/compose/docker-compose-fixturenet-laconic-console.yml rename to stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml diff --git a/app/data/compose/docker-compose-fixturenet-laconicd.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml similarity index 100% rename from app/data/compose/docker-compose-fixturenet-laconicd.yml rename to stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml diff --git a/app/data/compose/docker-compose-fixturenet-lotus.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-lotus.yml similarity index 100% rename from app/data/compose/docker-compose-fixturenet-lotus.yml rename to stack_orchestrator/data/compose/docker-compose-fixturenet-lotus.yml diff --git a/app/data/compose/docker-compose-fixturenet-optimism.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml similarity index 100% rename from app/data/compose/docker-compose-fixturenet-optimism.yml rename to stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml diff --git a/app/data/compose/docker-compose-fixturenet-plugeth.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-plugeth.yml similarity index 100% rename from app/data/compose/docker-compose-fixturenet-plugeth.yml rename to stack_orchestrator/data/compose/docker-compose-fixturenet-plugeth.yml diff --git a/app/data/compose/docker-compose-fixturenet-pocket.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-pocket.yml similarity index 100% rename from app/data/compose/docker-compose-fixturenet-pocket.yml rename to stack_orchestrator/data/compose/docker-compose-fixturenet-pocket.yml diff --git a/app/data/compose/docker-compose-fixturenet-sushiswap-subgraph-v3.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-sushiswap-subgraph-v3.yml similarity index 100% rename from app/data/compose/docker-compose-fixturenet-sushiswap-subgraph-v3.yml rename to stack_orchestrator/data/compose/docker-compose-fixturenet-sushiswap-subgraph-v3.yml diff --git a/app/data/compose/docker-compose-foundry.yml b/stack_orchestrator/data/compose/docker-compose-foundry.yml similarity index 100% rename from app/data/compose/docker-compose-foundry.yml rename to stack_orchestrator/data/compose/docker-compose-foundry.yml diff --git a/app/data/compose/docker-compose-go-ethereum-foundry.yml b/stack_orchestrator/data/compose/docker-compose-go-ethereum-foundry.yml similarity index 100% rename from app/data/compose/docker-compose-go-ethereum-foundry.yml rename to stack_orchestrator/data/compose/docker-compose-go-ethereum-foundry.yml diff --git a/app/data/compose/docker-compose-go-nitro.yml b/stack_orchestrator/data/compose/docker-compose-go-nitro.yml similarity index 60% rename from app/data/compose/docker-compose-go-nitro.yml rename to stack_orchestrator/data/compose/docker-compose-go-nitro.yml index 69b3afc8..552f621c 100644 --- a/app/data/compose/docker-compose-go-nitro.yml +++ b/stack_orchestrator/data/compose/docker-compose-go-nitro.yml @@ -11,10 +11,13 @@ services: condition: service_completed_successfully environment: CERC_NITRO_CHAIN_URL: ${CERC_NITRO_CHAIN_URL:-ws://fixturenet-eth-geth-1:8546} - CERC_NITRO_PK: ${CERC_NITRO_PK:-2d999770f7b5d49b694080f987b82bbc9fc9ac2b4dcc10b0f8aba7d700f69c6d} - CERC_NITRO_CHAIN_PK: ${CERC_NITRO_CHAIN_PK:-570b909da9669b2f35a0b1ac70b8358516d55ae1b5b3710e95e9a94395090597} + CERC_NITRO_PK: ${CERC_NITRO_PK:-f36f6dd450892224ee113899195ef922a4795d41c32cafb386d9aab6e0b7b0c6} + CERC_NITRO_CHAIN_PK: ${CERC_NITRO_CHAIN_PK:-888814df89c4358d7ddb3fa4b0213e7331239a80e1f013eaa7b2deca2a41a218} CERC_NITRO_USE_DURABLE_STORE: ${CERC_NITRO_USE_DURABLE_STORE:-true} CERC_NITRO_DURABLE_STORE_FOLDER: ${CERC_NITRO_DURABLE_STORE_FOLDER:-/app/data/nitro-store} + CERC_NITRO_MSG_PORT: ${CERC_NITRO_MSG_PORT:-3006} + CERC_NITRO_WS_MSG_PORT: ${CERC_NITRO_WS_MSG_PORT:-5006} + CERC_NITRO_RPC_PORT: ${CERC_NITRO_RPC_PORT:-4006} CERC_NA_ADDRESS: ${CERC_NA_ADDRESS} CERC_VPA_ADDRESS: ${CERC_VPA_ADDRESS} CERC_CA_ADDRESS: ${CERC_CA_ADDRESS} @@ -24,25 +27,15 @@ services: - nitro_deployment:/app/deployment - ../config/go-nitro/run-nitro-node.sh:/app/run-nitro-node.sh healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "4005"] - interval: 10s + test: ["CMD", "nc", "-vz", "localhost", "4006"] + interval: 30s timeout: 5s retries: 10 start_period: 10s ports: - - "3005" - - "4005:4005" - - "5005:5005" - - nitro-rpc-client: - image: cerc/nitro-rpc-client:local - hostname: nitro-rpc-client - restart: on-failure - depends_on: - # Wait for the go-nitro node to start - go-nitro: - condition: service_healthy - command: ["bash", "-c", "tail -f /dev/null"] + - "3006:3006" + - "4006:4006" + - "5006:5006" volumes: go_nitro_data: diff --git a/app/data/compose/docker-compose-graph-node.yml b/stack_orchestrator/data/compose/docker-compose-graph-node.yml similarity index 100% rename from app/data/compose/docker-compose-graph-node.yml rename to stack_orchestrator/data/compose/docker-compose-graph-node.yml diff --git a/app/data/compose/docker-compose-ipld-eth-beacon-db.yml b/stack_orchestrator/data/compose/docker-compose-ipld-eth-beacon-db.yml similarity index 100% rename from app/data/compose/docker-compose-ipld-eth-beacon-db.yml rename to stack_orchestrator/data/compose/docker-compose-ipld-eth-beacon-db.yml diff --git a/app/data/compose/docker-compose-ipld-eth-beacon-indexer.yml b/stack_orchestrator/data/compose/docker-compose-ipld-eth-beacon-indexer.yml similarity index 100% rename from app/data/compose/docker-compose-ipld-eth-beacon-indexer.yml rename to stack_orchestrator/data/compose/docker-compose-ipld-eth-beacon-indexer.yml diff --git a/app/data/compose/docker-compose-ipld-eth-db.yml b/stack_orchestrator/data/compose/docker-compose-ipld-eth-db.yml similarity index 100% rename from app/data/compose/docker-compose-ipld-eth-db.yml rename to stack_orchestrator/data/compose/docker-compose-ipld-eth-db.yml diff --git a/stack_orchestrator/data/compose/docker-compose-ipld-eth-server-payments.yml b/stack_orchestrator/data/compose/docker-compose-ipld-eth-server-payments.yml new file mode 100644 index 00000000..30cb94c2 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-ipld-eth-server-payments.yml @@ -0,0 +1,111 @@ +version: "3.2" +services: + ipld-eth-server-1: + restart: unless-stopped + depends_on: + ipld-eth-db: + condition: service_healthy + image: cerc/ipld-eth-server:local + environment: + SERVER_HTTP_PATH: 0.0.0.0:8081 + SERVER_GRAPHQL: "true" + SERVER_GRAPHQLPATH: 0.0.0.0:8082 + VDB_COMMAND: "serve" + ETH_CHAIN_CONFIG: "/tmp/chain.json" + DATABASE_NAME: cerc_testing + DATABASE_HOSTNAME: ipld-eth-db + DATABASE_PORT: 5432 + DATABASE_USER: "vdbm" + DATABASE_PASSWORD: "password" + ETH_CHAIN_ID: 99 + ETH_FORWARD_ETH_CALLS: "false" + ETH_FORWARD_GET_STORAGE_AT: "false" + ETH_PROXY_ON_ERROR: "false" + METRICS: "true" + PROM_HTTP: "true" + PROM_HTTP_ADDR: "0.0.0.0" + PROM_HTTP_PORT: "8090" + LOG_LEVEL: "debug" + CERC_REMOTE_DEBUG: ${CERC_REMOTE_DEBUG:-true} + NITRO_RUN_NODE_IN_PROCESS: ${CERC_NITRO_RUN_NODE_IN_PROCESS:-true} + NITRO_PK: ${CERC_NITRO_PK:-2d999770f7b5d49b694080f987b82bbc9fc9ac2b4dcc10b0f8aba7d700f69c6d} + NITRO_CHAIN_PK: ${CERC_NITRO_CHAIN_PK:-570b909da9669b2f35a0b1ac70b8358516d55ae1b5b3710e95e9a94395090597} + NITRO_CHAIN_URL: ${CERC_NITRO_CHAIN_URL:-ws://fixturenet-eth-geth-1:8546} + NITRO_USE_DURABLE_STORE: ${CERC_NITRO_USE_DURABLE_STORE:-true} + NITRO_DURABLE_STORE_FOLDER: ${CERC_NITRO_DURABLE_STORE_FOLDER:-/app/nitro-data/nitro-store} + CERC_NA_ADDRESS: ${CERC_NA_ADDRESS} + CERC_VPA_ADDRESS: ${CERC_VPA_ADDRESS} + CERC_CA_ADDRESS: ${CERC_CA_ADDRESS} + entrypoint: ["bash", "-c", "/app/entrypoint.sh"] + volumes: + - type: bind + source: ../config/ipld-eth-server/chain.json + target: /tmp/chain.json + - eth_server_nitro_data:/app/nitro-data + - nitro_deployment:/app/deployment + - ../config/ipld-eth-server/entrypoint.sh:/app/entrypoint.sh + ports: + - "8081" + - "8082" + - "8090" + - "40000" + - "3005:3005" + - "4005:4005" + - "5005:5005" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "8081"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + + ipld-eth-server-2: + restart: unless-stopped + depends_on: + ipld-eth-db: + condition: service_healthy + image: cerc/ipld-eth-server:local + environment: + SERVER_HTTP_PATH: 0.0.0.0:8081 + SERVER_GRAPHQL: "true" + SERVER_GRAPHQLPATH: 0.0.0.0:8082 + VDB_COMMAND: "serve" + ETH_CHAIN_CONFIG: "/tmp/chain.json" + DATABASE_NAME: cerc_testing + DATABASE_HOSTNAME: ipld-eth-db + DATABASE_PORT: 5432 + DATABASE_USER: "vdbm" + DATABASE_PASSWORD: "password" + ETH_CHAIN_ID: 99 + ETH_FORWARD_ETH_CALLS: "false" + ETH_FORWARD_GET_STORAGE_AT: "false" + ETH_PROXY_ON_ERROR: "false" + METRICS: "true" + PROM_HTTP: "true" + PROM_HTTP_ADDR: "0.0.0.0" + PROM_HTTP_PORT: "8090" + LOG_LEVEL: "debug" + CERC_REMOTE_DEBUG: ${CERC_REMOTE_DEBUG:-true} + NITRO_RUN_NODE_IN_PROCESS: ${CERC_NITRO_RUN_NODE_IN_PROCESS:-false} + NITRO_ENDPOINT: ${CERC_NITRO_ENDPOINT:-go-nitro:4006/api/v1} + entrypoint: ["bash", "-c", "/app/entrypoint.sh"] + volumes: + - type: bind + source: ../config/ipld-eth-server/chain.json + target: /tmp/chain.json + - ../config/ipld-eth-server/entrypoint.sh:/app/entrypoint.sh + ports: + - "8081" + - "8082" + - "8090" + - "40000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "8081"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + +volumes: + eth_server_nitro_data: + nitro_deployment: diff --git a/app/data/compose/docker-compose-ipld-eth-server.yml b/stack_orchestrator/data/compose/docker-compose-ipld-eth-server.yml similarity index 100% rename from app/data/compose/docker-compose-ipld-eth-server.yml rename to stack_orchestrator/data/compose/docker-compose-ipld-eth-server.yml diff --git a/app/data/compose/docker-compose-keycloak.yml b/stack_orchestrator/data/compose/docker-compose-keycloak.yml similarity index 100% rename from app/data/compose/docker-compose-keycloak.yml rename to stack_orchestrator/data/compose/docker-compose-keycloak.yml diff --git a/app/data/compose/docker-compose-kubo.yml b/stack_orchestrator/data/compose/docker-compose-kubo.yml similarity index 100% rename from app/data/compose/docker-compose-kubo.yml rename to stack_orchestrator/data/compose/docker-compose-kubo.yml diff --git a/stack_orchestrator/data/compose/docker-compose-laconic-dot-com.yml b/stack_orchestrator/data/compose/docker-compose-laconic-dot-com.yml new file mode 100644 index 00000000..7ee35e2b --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-laconic-dot-com.yml @@ -0,0 +1,8 @@ +version: "3.2" + +services: + laconic-dot-com: + image: cerc/laconic-dot-com:local + restart: always + ports: + - "3000" diff --git a/app/data/compose/docker-compose-laconicd.yml b/stack_orchestrator/data/compose/docker-compose-laconicd.yml similarity index 100% rename from app/data/compose/docker-compose-laconicd.yml rename to stack_orchestrator/data/compose/docker-compose-laconicd.yml diff --git a/app/data/compose/docker-compose-lasso.yml b/stack_orchestrator/data/compose/docker-compose-lasso.yml similarity index 100% rename from app/data/compose/docker-compose-lasso.yml rename to stack_orchestrator/data/compose/docker-compose-lasso.yml diff --git a/app/data/compose/docker-compose-mainnet-eth-api-proxy.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-api-proxy.yml similarity index 100% rename from app/data/compose/docker-compose-mainnet-eth-api-proxy.yml rename to stack_orchestrator/data/compose/docker-compose-mainnet-eth-api-proxy.yml diff --git a/stack_orchestrator/data/compose/docker-compose-mainnet-eth-ipld-eth-db.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-ipld-eth-db.yml new file mode 100644 index 00000000..49cc2de3 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-ipld-eth-db.yml @@ -0,0 +1,29 @@ +version: "3.2" + +services: + migrations: + restart: on-failure + depends_on: + ipld-eth-db: + condition: service_healthy + image: cerc/ipld-eth-db:local + env_file: + - ../config/mainnet-eth-ipld-eth-db/db.env + + ipld-eth-db: + image: timescale/timescaledb:2.8.1-pg14 + restart: always + env_file: + - ../config/mainnet-eth-ipld-eth-db/db.env + volumes: + - mainnet_eth_ipld_eth_db:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 3s + ports: + - "5432" +volumes: + mainnet_eth_ipld_eth_db: diff --git a/stack_orchestrator/data/compose/docker-compose-mainnet-eth-ipld-eth-server.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-ipld-eth-server.yml new file mode 100644 index 00000000..4341c6a1 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-ipld-eth-server.yml @@ -0,0 +1,24 @@ +version: "3.7" +services: + ipld-eth-server: + restart: always + depends_on: + ipld-eth-db: + condition: service_healthy + image: cerc/ipld-eth-server:local + env_file: + - ../config/mainnet-eth-ipld-eth-db/db.env + - ../config/mainnet-eth-ipld-eth-server/srv.env + volumes: + - ../config/mainnet-eth-ipld-eth-server/config.toml:/app/config.toml:ro + ports: + - "8081" + - "8082" + - "8090" + - "40001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "8081"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s diff --git a/app/data/compose/docker-compose-mainnet-eth-keycloak.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-keycloak.yml similarity index 94% rename from app/data/compose/docker-compose-mainnet-eth-keycloak.yml rename to stack_orchestrator/data/compose/docker-compose-mainnet-eth-keycloak.yml index dfa9a804..1674c62e 100644 --- a/app/data/compose/docker-compose-mainnet-eth-keycloak.yml +++ b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-keycloak.yml @@ -6,7 +6,7 @@ services: env_file: - ../config/mainnet-eth-keycloak/keycloak.env healthcheck: - test: ["CMD", "nc", "-v", "localhost", "5432"] + test: ["CMD", "nc", "-v", "localhost", "35432"] interval: 30s timeout: 10s retries: 10 @@ -14,7 +14,7 @@ services: volumes: - mainnet_eth_keycloak_db:/var/lib/postgresql/data ports: - - 5432 + - 35432 keycloak: image: cerc/keycloak:local diff --git a/app/data/compose/docker-compose-mainnet-eth-metrics.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-metrics.yml similarity index 100% rename from app/data/compose/docker-compose-mainnet-eth-metrics.yml rename to stack_orchestrator/data/compose/docker-compose-mainnet-eth-metrics.yml diff --git a/stack_orchestrator/data/compose/docker-compose-mainnet-eth-plugeth.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-plugeth.yml new file mode 100644 index 00000000..a8b301d2 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-mainnet-eth-plugeth.yml @@ -0,0 +1,72 @@ + +services: + + mainnet-eth-geth-1: + restart: always + hostname: mainnet-eth-geth-1 + cap_add: + - SYS_PTRACE + image: cerc/plugeth-with-plugins:local + entrypoint: /bin/sh + command: -c "/opt/run-geth.sh" + env_file: + - ../config/mainnet-eth-ipld-eth-db/db.env + - ../config/mainnet-eth-plugeth/geth.env + volumes: + - mainnet_eth_plugeth_geth_1_data:/data + - mainnet_eth_plugeth_config_data:/etc/mainnet-eth + - ../config/mainnet-eth-plugeth/scripts/run-geth.sh:/opt/run-geth.sh + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "8545"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 3s + ports: + # http api + - "8545" + # ws api + - "8546" + # ws el + - "8551" + # p2p + - "30303" + - "30303/udp" + # debugging + - "40000" + # metrics + - "6060" + + mainnet-eth-lighthouse-1: + restart: always + hostname: mainnet-eth-lighthouse-1 + healthcheck: + test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:5052/eth/v2/beacon/blocks/head"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 30s + environment: + LIGHTHOUSE_EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551" + env_file: + - ../config/mainnet-eth-plugeth/lighthouse.env + image: cerc/lighthouse:local + entrypoint: /bin/sh + command: -c "/opt/run-lighthouse.sh" + volumes: + - mainnet_eth_plugeth_lighthouse_1_data:/data + - mainnet_eth_plugeth_config_data:/etc/mainnet-eth + - ../config/mainnet-eth-plugeth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh + ports: + # api + - "5052" + # metrics + - "5054" + # p2p + - "9000" + - "9000/udp" + +volumes: + mainnet_eth_plugeth_config_data: + mainnet_eth_plugeth_geth_1_data: + mainnet_eth_plugeth_lighthouse_1_data: diff --git a/app/data/compose/docker-compose-mainnet-eth.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-eth.yml similarity index 100% rename from app/data/compose/docker-compose-mainnet-eth.yml rename to stack_orchestrator/data/compose/docker-compose-mainnet-eth.yml diff --git a/app/data/compose/docker-compose-mainnet-go-opera.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-go-opera.yml similarity index 100% rename from app/data/compose/docker-compose-mainnet-go-opera.yml rename to stack_orchestrator/data/compose/docker-compose-mainnet-go-opera.yml diff --git a/app/data/compose/docker-compose-mainnet-laconicd.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-laconicd.yml similarity index 100% rename from app/data/compose/docker-compose-mainnet-laconicd.yml rename to stack_orchestrator/data/compose/docker-compose-mainnet-laconicd.yml diff --git a/app/data/compose/docker-compose-mobymask-app-v3.yml b/stack_orchestrator/data/compose/docker-compose-mobymask-app-v3.yml similarity index 100% rename from app/data/compose/docker-compose-mobymask-app-v3.yml rename to stack_orchestrator/data/compose/docker-compose-mobymask-app-v3.yml diff --git a/app/data/compose/docker-compose-mobymask-app.yml b/stack_orchestrator/data/compose/docker-compose-mobymask-app.yml similarity index 100% rename from app/data/compose/docker-compose-mobymask-app.yml rename to stack_orchestrator/data/compose/docker-compose-mobymask-app.yml diff --git a/app/data/compose/docker-compose-mobymask-snap.yml b/stack_orchestrator/data/compose/docker-compose-mobymask-snap.yml similarity index 100% rename from app/data/compose/docker-compose-mobymask-snap.yml rename to stack_orchestrator/data/compose/docker-compose-mobymask-snap.yml diff --git a/app/data/compose/docker-compose-nitro-contracts.yml b/stack_orchestrator/data/compose/docker-compose-nitro-contracts.yml similarity index 100% rename from app/data/compose/docker-compose-nitro-contracts.yml rename to stack_orchestrator/data/compose/docker-compose-nitro-contracts.yml diff --git a/stack_orchestrator/data/compose/docker-compose-nitro-rpc-client.yml b/stack_orchestrator/data/compose/docker-compose-nitro-rpc-client.yml new file mode 100644 index 00000000..ae73de62 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-nitro-rpc-client.yml @@ -0,0 +1,7 @@ +version: '3.7' +services: + nitro-rpc-client: + image: cerc/nitro-rpc-client:local + hostname: nitro-rpc-client + restart: on-failure + command: ["bash", "-c", "tail -f /dev/null"] diff --git a/app/data/compose/docker-compose-peer-test-app.yml b/stack_orchestrator/data/compose/docker-compose-peer-test-app.yml similarity index 100% rename from app/data/compose/docker-compose-peer-test-app.yml rename to stack_orchestrator/data/compose/docker-compose-peer-test-app.yml diff --git a/stack_orchestrator/data/compose/docker-compose-ponder-indexer.yml b/stack_orchestrator/data/compose/docker-compose-ponder-indexer.yml new file mode 100644 index 00000000..233ecbd7 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-ponder-indexer.yml @@ -0,0 +1,80 @@ +version: '3.7' + +services: + ponder-er20-contracts: + image: cerc/watcher-erc20:local + restart: on-failure + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-http://fixturenet-eth-geth-1:8545} + CERC_PRIVATE_KEY_DEPLOYER: ${CERC_PRIVATE_KEY_DEPLOYER:-0x888814df89c4358d7ddb3fa4b0213e7331239a80e1f013eaa7b2deca2a41a218} + volumes: + - ../config/ponder/deploy-erc20-contract.sh:/app/deploy-erc20-contract.sh + - erc20_deployment:/app/deployment + command: ["bash", "-c", "/app/deploy-erc20-contract.sh"] + extra_hosts: + - "host.docker.internal:host-gateway" + + ponder-app-indexer-1: + hostname: ponder-app-indexer-1 + restart: unless-stopped + image: cerc/ponder:local + working_dir: /app/examples/token-erc20 + environment: + CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99} + CERC_PONDER_RPC_URL_1: ${PONDER_RPC_URL_1:-http://ipld-eth-server-2:8081} + CERC_PONDER_NITRO_PK: ${CERC_PONDER_INDEXER_NITRO_PK_1:-58368d20ff12f17669c06158c21d885897aa56f9be430edc789614bf9851d53f} + CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_INDEXER_NITRO_CHAIN_PK_1:-fb1e9af328c283ca3e2486e7c24d13582b7912057d8b9542ff41503c85bc05c0} + CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546} + CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR} + CERC_UPSTREAM_NITRO_ADDRESS: ${CERC_UPSTREAM_NITRO_ADDRESS:-0x660a4bEF3fbC863Fcd8D3CDB39242aE513d7D92e} + CERC_UPSTREAM_NITRO_MULTIADDR: ${CERC_UPSTREAM_NITRO_MULTIADDR:-/dns4/go-nitro/tcp/5006/ws/p2p/16Uiu2HAmNUiX7bpCpbo5JdqEebp85ptGU2Vk2AT9E3BykvbwQ3F9} + CERC_UPSTREAM_NITRO_PAY_AMOUNT: ${CERC_UPSTREAM_NITRO_PAY_AMOUNT:-100} + command: ["bash", "./ponder-start.sh"] + volumes: + - ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh + - ../config/ponder/ponder.indexer-1.config.ts:/app/examples/token-erc20/ponder.config.ts + - ../config/ponder/base-rates-config.json:/app/examples/token-erc20/base-rates-config.json + - peers_ids:/peers + - nitro_deployment:/nitro + - erc20_deployment:/erc20 + - ponder_indexer_1_nitro_data:/app/examples/token-erc20/.ponder/nitro-db + ports: + - "127.0.0.1:42070:42070" + extra_hosts: + - "host.docker.internal:host-gateway" + + ponder-app-indexer-2: + hostname: ponder-app-indexer-2 + restart: unless-stopped + image: cerc/ponder:local + working_dir: /app/examples/token-erc20 + environment: + CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99} + CERC_INDEXER_GQL_ENDPOINT: ${CERC_INDEXER_GQL_ENDPOINT:-http://ponder-app-indexer-1:42070/graphql} + CERC_PONDER_NITRO_PK: ${CERC_PONDER_INDEXER_NITRO_PK_2:-0aca28ba64679f63d71e671ab4dbb32aaa212d4789988e6ca47da47601c18fe2} + CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_INDEXER_NITRO_CHAIN_PK_2:-6177345b77c4069ac4d553f8b43cf68a799ca4bb63eac93d6cf796d63694ebf0} + CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546} + CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR} + CERC_INDEXER_NITRO_ADDRESS: ${CERC_INDEXER_NITRO_ADDRESS:-0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d} + CERC_UPSTREAM_NITRO_PAY_AMOUNT: ${CERC_UPSTREAM_NITRO_PAY_AMOUNT:-100} + command: ["bash", "./ponder-start.sh"] + volumes: + - ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh + - ../config/ponder/ponder.indexer-2.config.ts:/app/examples/token-erc20/ponder.config.ts + - ../config/ponder/base-rates-config.json:/app/examples/token-erc20/base-rates-config.json + - peers_ids:/peers + - nitro_deployment:/nitro + - erc20_deployment:/erc20 + - ponder_indexer_2_nitro_data:/app/examples/token-erc20/.ponder/nitro-db + ports: + - "127.0.0.1:42071:42070" + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + peers_ids: + nitro_deployment: + erc20_deployment: + ponder_indexer_1_nitro_data: + ponder_indexer_2_nitro_data: diff --git a/stack_orchestrator/data/compose/docker-compose-ponder-watcher.yml b/stack_orchestrator/data/compose/docker-compose-ponder-watcher.yml new file mode 100644 index 00000000..030782d7 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-ponder-watcher.yml @@ -0,0 +1,39 @@ + +version: '3.7' + +services: + ponder-app-watcher: + hostname: ponder-app-watcher + depends_on: + - ponder-app-indexer-1 + restart: unless-stopped + image: cerc/ponder:local + working_dir: /app/examples/token-erc20 + environment: + CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99} + CERC_PONDER_NITRO_PK: ${CERC_PONDER_WATCHER_NITRO_PK:-febb3b74b0b52d0976f6571d555f4ac8b91c308dfa25c7b58d1e6a7c3f50c781} + CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_WATCHER_NITRO_CHAIN_PK:-be4aa664815ea3bc3d63118649a733f6c96b243744310806ecb6d96359ab62cf} + CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546} + CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR} + CERC_INDEXER_GQL_ENDPOINT: ${CERC_INDEXER_GQL_ENDPOINT:-http://ponder-app-indexer-2:42070/graphql} + CERC_INDEXER_NITRO_ADDRESS: ${CERC_INDEXER_NITRO_ADDRESS:-0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01} + CERC_INDEXER_NITRO_PAY_AMOUNT: ${CERC_INDEXER_NITRO_PAY_AMOUNT:-50} + command: ["bash", "./ponder-start.sh"] + volumes: + - ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh + - ../config/ponder/ponder.watcher.config.ts:/app/examples/token-erc20/ponder.config.ts + - ../config/ponder/base-rates-config.json:/app/examples/token-erc20/base-rates-config.json + - peers_ids:/peers + - nitro_deployment:/nitro + - erc20_deployment:/erc20 + - ponder_watcher_nitro_data:/app/examples/token-erc20/.ponder/nitro-db + ports: + - "127.0.0.1:42069:42069" + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + peers_ids: + nitro_deployment: + erc20_deployment: + ponder_watcher_nitro_data: diff --git a/app/data/compose/docker-compose-reth.yml b/stack_orchestrator/data/compose/docker-compose-reth.yml similarity index 100% rename from app/data/compose/docker-compose-reth.yml rename to stack_orchestrator/data/compose/docker-compose-reth.yml diff --git a/app/data/compose/docker-compose-sushiswap-subgraph-v3.yml b/stack_orchestrator/data/compose/docker-compose-sushiswap-subgraph-v3.yml similarity index 100% rename from app/data/compose/docker-compose-sushiswap-subgraph-v3.yml rename to stack_orchestrator/data/compose/docker-compose-sushiswap-subgraph-v3.yml diff --git a/app/data/compose/docker-compose-test.yml b/stack_orchestrator/data/compose/docker-compose-test.yml similarity index 100% rename from app/data/compose/docker-compose-test.yml rename to stack_orchestrator/data/compose/docker-compose-test.yml diff --git a/app/data/compose/docker-compose-tx-spammer.yml b/stack_orchestrator/data/compose/docker-compose-tx-spammer.yml similarity index 100% rename from app/data/compose/docker-compose-tx-spammer.yml rename to stack_orchestrator/data/compose/docker-compose-tx-spammer.yml diff --git a/app/data/compose/docker-compose-watcher-azimuth.yml b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml similarity index 100% rename from app/data/compose/docker-compose-watcher-azimuth.yml rename to stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml diff --git a/app/data/compose/docker-compose-watcher-erc20.yml b/stack_orchestrator/data/compose/docker-compose-watcher-erc20.yml similarity index 92% rename from app/data/compose/docker-compose-watcher-erc20.yml rename to stack_orchestrator/data/compose/docker-compose-watcher-erc20.yml index a094250c..fe217e18 100644 --- a/app/data/compose/docker-compose-watcher-erc20.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-erc20.yml @@ -34,7 +34,7 @@ services: - ETH_RPC_URL=http://go-ethereum:8545 command: ["sh", "-c", "yarn server"] volumes: - - ../config/watcher-erc20/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml + - ../config/watcher-erc20/erc20-watcher.toml:/app/environments/local.toml ports: - "0.0.0.0:3002:3001" - "0.0.0.0:9002:9001" diff --git a/app/data/compose/docker-compose-watcher-erc721.yml b/stack_orchestrator/data/compose/docker-compose-watcher-erc721.yml similarity index 100% rename from app/data/compose/docker-compose-watcher-erc721.yml rename to stack_orchestrator/data/compose/docker-compose-watcher-erc721.yml diff --git a/app/data/compose/docker-compose-watcher-gelato.yml b/stack_orchestrator/data/compose/docker-compose-watcher-gelato.yml similarity index 100% rename from app/data/compose/docker-compose-watcher-gelato.yml rename to stack_orchestrator/data/compose/docker-compose-watcher-gelato.yml diff --git a/app/data/compose/docker-compose-watcher-mobymask-v2.yml b/stack_orchestrator/data/compose/docker-compose-watcher-mobymask-v2.yml similarity index 100% rename from app/data/compose/docker-compose-watcher-mobymask-v2.yml rename to stack_orchestrator/data/compose/docker-compose-watcher-mobymask-v2.yml diff --git a/app/data/compose/docker-compose-watcher-mobymask-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-mobymask-v3.yml similarity index 97% rename from app/data/compose/docker-compose-watcher-mobymask-v3.yml rename to stack_orchestrator/data/compose/docker-compose-watcher-mobymask-v3.yml index bd07d7ef..075c34e6 100644 --- a/app/data/compose/docker-compose-watcher-mobymask-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-mobymask-v3.yml @@ -89,8 +89,8 @@ services: CERC_PEER_ID: ${CERC_PEER_ID} CERC_ENABLE_UPSTREAM_PAYMENTS: ${CERC_ENABLE_UPSTREAM_PAYMENTS} CERC_UPSTREAM_NITRO_ADDRESS: ${CERC_UPSTREAM_NITRO_ADDRESS:-0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE} - CERC_UPSTREAM_NITRO_MULTIADDR: ${CERC_UPSTREAM_NITRO_MULTIADDR:-/dns4/go-nitro/tcp/5005/ws/p2p/16Uiu2HAmSjXJqsyBJgcBUU2HQmykxGseafSatbpq5471XmuaUqyv} - CERC_UPSTREAM_NITRO_PAY_AMOUNT: ${CERC_UPSTREAM_NITRO_PAY_AMOUNT:-5000} + CERC_UPSTREAM_NITRO_MULTIADDR: ${CERC_UPSTREAM_NITRO_MULTIADDR:-/dns4/ipld-eth-server-1/tcp/5005/ws/p2p/16Uiu2HAmSjXJqsyBJgcBUU2HQmykxGseafSatbpq5471XmuaUqyv} + CERC_UPSTREAM_NITRO_PAY_AMOUNT: ${CERC_UPSTREAM_NITRO_PAY_AMOUNT:-100} command: ["bash", "./start-server.sh"] volumes: - ../config/watcher-mobymask-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml diff --git a/app/data/compose/docker-compose-watcher-mobymask.yml b/stack_orchestrator/data/compose/docker-compose-watcher-mobymask.yml similarity index 100% rename from app/data/compose/docker-compose-watcher-mobymask.yml rename to stack_orchestrator/data/compose/docker-compose-watcher-mobymask.yml diff --git a/app/data/compose/docker-compose-watcher-sushiswap.yml b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap.yml similarity index 100% rename from app/data/compose/docker-compose-watcher-sushiswap.yml rename to stack_orchestrator/data/compose/docker-compose-watcher-sushiswap.yml diff --git a/app/data/compose/docker-compose-watcher-uniswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-uniswap-v3.yml similarity index 100% rename from app/data/compose/docker-compose-watcher-uniswap-v3.yml rename to stack_orchestrator/data/compose/docker-compose-watcher-uniswap-v3.yml diff --git a/app/data/config/contract-sushiswap/deploy-core-contracts.sh b/stack_orchestrator/data/config/contract-sushiswap/deploy-core-contracts.sh similarity index 100% rename from app/data/config/contract-sushiswap/deploy-core-contracts.sh rename to stack_orchestrator/data/config/contract-sushiswap/deploy-core-contracts.sh diff --git a/app/data/config/contract-sushiswap/deploy-periphery-contracts.sh b/stack_orchestrator/data/config/contract-sushiswap/deploy-periphery-contracts.sh similarity index 100% rename from app/data/config/contract-sushiswap/deploy-periphery-contracts.sh rename to stack_orchestrator/data/config/contract-sushiswap/deploy-periphery-contracts.sh diff --git a/app/data/config/contract-sushiswap/deployment-params.env b/stack_orchestrator/data/config/contract-sushiswap/deployment-params.env similarity index 100% rename from app/data/config/contract-sushiswap/deployment-params.env rename to stack_orchestrator/data/config/contract-sushiswap/deployment-params.env diff --git a/app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json b/stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json similarity index 100% rename from app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json rename to stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json diff --git a/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml b/stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml similarity index 100% rename from app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml rename to stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml diff --git a/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml b/stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml similarity index 100% rename from app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml rename to stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml diff --git a/app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml b/stack_orchestrator/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml similarity index 100% rename from app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml rename to stack_orchestrator/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml diff --git a/app/data/config/fixturenet-eth/fixturenet-eth.env b/stack_orchestrator/data/config/fixturenet-eth/fixturenet-eth.env similarity index 100% rename from app/data/config/fixturenet-eth/fixturenet-eth.env rename to stack_orchestrator/data/config/fixturenet-eth/fixturenet-eth.env diff --git a/app/data/config/fixturenet-laconicd/create-fixturenet.sh b/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh similarity index 100% rename from app/data/config/fixturenet-laconicd/create-fixturenet.sh rename to stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh diff --git a/app/data/config/fixturenet-laconicd/export-myaddress.sh b/stack_orchestrator/data/config/fixturenet-laconicd/export-myaddress.sh similarity index 100% rename from app/data/config/fixturenet-laconicd/export-myaddress.sh rename to stack_orchestrator/data/config/fixturenet-laconicd/export-myaddress.sh diff --git a/app/data/config/fixturenet-laconicd/export-mykey.sh b/stack_orchestrator/data/config/fixturenet-laconicd/export-mykey.sh similarity index 100% rename from app/data/config/fixturenet-laconicd/export-mykey.sh rename to stack_orchestrator/data/config/fixturenet-laconicd/export-mykey.sh diff --git a/app/data/config/mainnet-laconicd/registry-cli-config-template.yml b/stack_orchestrator/data/config/fixturenet-laconicd/registry-cli-config-template.yml similarity index 92% rename from app/data/config/mainnet-laconicd/registry-cli-config-template.yml rename to stack_orchestrator/data/config/fixturenet-laconicd/registry-cli-config-template.yml index 16432c18..d558956a 100644 --- a/app/data/config/mainnet-laconicd/registry-cli-config-template.yml +++ b/stack_orchestrator/data/config/fixturenet-laconicd/registry-cli-config-template.yml @@ -5,5 +5,5 @@ services: userKey: REPLACE_WITH_MYKEY bondId: chainId: laconic_9000-1 - gas: 250000 + gas: 350000 fees: 200000aphoton diff --git a/app/data/config/fixturenet-lotus/fund-account.sh b/stack_orchestrator/data/config/fixturenet-lotus/fund-account.sh similarity index 100% rename from app/data/config/fixturenet-lotus/fund-account.sh rename to stack_orchestrator/data/config/fixturenet-lotus/fund-account.sh diff --git a/app/data/config/fixturenet-lotus/lotus-env.env b/stack_orchestrator/data/config/fixturenet-lotus/lotus-env.env similarity index 100% rename from app/data/config/fixturenet-lotus/lotus-env.env rename to stack_orchestrator/data/config/fixturenet-lotus/lotus-env.env diff --git a/app/data/config/fixturenet-lotus/setup-miner.sh b/stack_orchestrator/data/config/fixturenet-lotus/setup-miner.sh similarity index 100% rename from app/data/config/fixturenet-lotus/setup-miner.sh rename to stack_orchestrator/data/config/fixturenet-lotus/setup-miner.sh diff --git a/app/data/config/fixturenet-lotus/setup-node.sh b/stack_orchestrator/data/config/fixturenet-lotus/setup-node.sh similarity index 100% rename from app/data/config/fixturenet-lotus/setup-node.sh rename to stack_orchestrator/data/config/fixturenet-lotus/setup-node.sh diff --git a/app/data/config/fixturenet-optimism/generate-l2-config.sh b/stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh similarity index 100% rename from app/data/config/fixturenet-optimism/generate-l2-config.sh rename to stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh diff --git a/app/data/config/fixturenet-optimism/l1-params.env b/stack_orchestrator/data/config/fixturenet-optimism/l1-params.env similarity index 100% rename from app/data/config/fixturenet-optimism/l1-params.env rename to stack_orchestrator/data/config/fixturenet-optimism/l1-params.env diff --git a/app/data/config/fixturenet-optimism/optimism-contracts/run.sh b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh similarity index 100% rename from app/data/config/fixturenet-optimism/optimism-contracts/run.sh rename to stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh diff --git a/app/data/config/fixturenet-optimism/optimism-contracts/update-config.js b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js similarity index 100% rename from app/data/config/fixturenet-optimism/optimism-contracts/update-config.js rename to stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js diff --git a/app/data/config/fixturenet-optimism/run-op-batcher.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh similarity index 100% rename from app/data/config/fixturenet-optimism/run-op-batcher.sh rename to stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh diff --git a/app/data/config/fixturenet-optimism/run-op-geth.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh similarity index 100% rename from app/data/config/fixturenet-optimism/run-op-geth.sh rename to stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh diff --git a/app/data/config/fixturenet-optimism/run-op-node.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh similarity index 100% rename from app/data/config/fixturenet-optimism/run-op-node.sh rename to stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh diff --git a/app/data/config/fixturenet-optimism/run-op-proposer.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh similarity index 100% rename from app/data/config/fixturenet-optimism/run-op-proposer.sh rename to stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh diff --git a/app/data/config/fixturenet-pocket/chains.json b/stack_orchestrator/data/config/fixturenet-pocket/chains.json similarity index 100% rename from app/data/config/fixturenet-pocket/chains.json rename to stack_orchestrator/data/config/fixturenet-pocket/chains.json diff --git a/app/data/config/fixturenet-pocket/create-fixturenet.sh b/stack_orchestrator/data/config/fixturenet-pocket/create-fixturenet.sh similarity index 100% rename from app/data/config/fixturenet-pocket/create-fixturenet.sh rename to stack_orchestrator/data/config/fixturenet-pocket/create-fixturenet.sh diff --git a/app/data/config/fixturenet-pocket/genesis.json b/stack_orchestrator/data/config/fixturenet-pocket/genesis.json similarity index 100% rename from app/data/config/fixturenet-pocket/genesis.json rename to stack_orchestrator/data/config/fixturenet-pocket/genesis.json diff --git a/app/data/config/fixturenet-sushiswap-subgraph-v3/lotus-fixturenet.js.template b/stack_orchestrator/data/config/fixturenet-sushiswap-subgraph-v3/lotus-fixturenet.js.template similarity index 100% rename from app/data/config/fixturenet-sushiswap-subgraph-v3/lotus-fixturenet.js.template rename to stack_orchestrator/data/config/fixturenet-sushiswap-subgraph-v3/lotus-fixturenet.js.template diff --git a/app/data/config/fixturenet-sushiswap-subgraph-v3/run-blocks.sh b/stack_orchestrator/data/config/fixturenet-sushiswap-subgraph-v3/run-blocks.sh similarity index 100% rename from app/data/config/fixturenet-sushiswap-subgraph-v3/run-blocks.sh rename to stack_orchestrator/data/config/fixturenet-sushiswap-subgraph-v3/run-blocks.sh diff --git a/app/data/config/fixturenet-sushiswap-subgraph-v3/run-v3.sh b/stack_orchestrator/data/config/fixturenet-sushiswap-subgraph-v3/run-v3.sh similarity index 100% rename from app/data/config/fixturenet-sushiswap-subgraph-v3/run-v3.sh rename to stack_orchestrator/data/config/fixturenet-sushiswap-subgraph-v3/run-v3.sh diff --git a/app/data/config/foundry/foundry.toml b/stack_orchestrator/data/config/foundry/foundry.toml similarity index 100% rename from app/data/config/foundry/foundry.toml rename to stack_orchestrator/data/config/foundry/foundry.toml diff --git a/app/data/config/go-nitro/run-nitro-node.sh b/stack_orchestrator/data/config/go-nitro/run-nitro-node.sh similarity index 64% rename from app/data/config/go-nitro/run-nitro-node.sh rename to stack_orchestrator/data/config/go-nitro/run-nitro-node.sh index edfb1cdf..249937ef 100755 --- a/app/data/config/go-nitro/run-nitro-node.sh +++ b/stack_orchestrator/data/config/go-nitro/run-nitro-node.sh @@ -29,12 +29,13 @@ fi echo "Running Nitro node" +# Assuming CERC_NITRO_CHAIN_URL is of format ://host:port +ws_host=$(echo "$CERC_NITRO_CHAIN_URL" | awk -F '://' '{print $2}' | cut -d ':' -f 1) +ws_port=$(echo "$CERC_NITRO_CHAIN_URL" | awk -F '://' '{print $2}' | cut -d ':' -f 2) + # Wait till chain endpoint is available retry_interval=5 while true; do - # Assuming CERC_NITRO_CHAIN_URL is of format ://host:port - ws_host=$(echo "$CERC_NITRO_CHAIN_URL" | awk -F '://' '{print $2}' | cut -d ':' -f 1) - ws_port=$(echo "$CERC_NITRO_CHAIN_URL" | awk -F '://' '{print $2}' | cut -d ':' -f 2) nc -z -w 1 "$ws_host" "$ws_port" if [ $? -eq 0 ]; then @@ -46,4 +47,4 @@ while true; do sleep $retry_interval done -./nitro -chainurl ${CERC_NITRO_CHAIN_URL} -msgport 3005 -rpcport 4005 -wsmsgport 5005 -pk ${CERC_NITRO_PK} -chainpk ${CERC_NITRO_CHAIN_PK} -naaddress ${NA_ADDRESS} -vpaaddress ${VPA_ADDRESS} -caaddress ${CA_ADDRESS} -usedurablestore ${CERC_NITRO_USE_DURABLE_STORE} -durablestorefolder ${CERC_NITRO_DURABLE_STORE_FOLDER} +./nitro -chainurl ${CERC_NITRO_CHAIN_URL} -msgport ${CERC_NITRO_MSG_PORT} -rpcport ${CERC_NITRO_RPC_PORT} -wsmsgport ${CERC_NITRO_WS_MSG_PORT} -publicip "0.0.0.0" -pk ${CERC_NITRO_PK} -chainpk ${CERC_NITRO_CHAIN_PK} -naaddress ${NA_ADDRESS} -vpaaddress ${VPA_ADDRESS} -caaddress ${CA_ADDRESS} -usedurablestore ${CERC_NITRO_USE_DURABLE_STORE} -durablestorefolder ${CERC_NITRO_DURABLE_STORE_FOLDER} -tlscertfilepath "" -tlsKeyFilepath "" diff --git a/app/data/config/ipld-eth-beacon-indexer/indexer.env b/stack_orchestrator/data/config/ipld-eth-beacon-indexer/indexer.env similarity index 100% rename from app/data/config/ipld-eth-beacon-indexer/indexer.env rename to stack_orchestrator/data/config/ipld-eth-beacon-indexer/indexer.env diff --git a/app/data/config/ipld-eth-server/chain.json b/stack_orchestrator/data/config/ipld-eth-server/chain.json similarity index 100% rename from app/data/config/ipld-eth-server/chain.json rename to stack_orchestrator/data/config/ipld-eth-server/chain.json diff --git a/stack_orchestrator/data/config/ipld-eth-server/entrypoint.sh b/stack_orchestrator/data/config/ipld-eth-server/entrypoint.sh new file mode 100755 index 00000000..37e232c6 --- /dev/null +++ b/stack_orchestrator/data/config/ipld-eth-server/entrypoint.sh @@ -0,0 +1,71 @@ +#!/bin/sh + +read_nitro_addresses() { + nitro_addresses_file="/app/deployment/nitro-addresses.json" + + # Check if CERC_NA_ADDRESS environment variable is set + if [ -n "$CERC_NA_ADDRESS" ]; then + echo "CERC_NA_ADDRESS is set to '$CERC_NA_ADDRESS'" + echo "CERC_VPA_ADDRESS is set to '$CERC_VPA_ADDRESS'" + echo "CERC_CA_ADDRESS is set to '$CERC_CA_ADDRESS'" + echo "Using the above Nitro addresses" + + export NITRO_NA_ADDRESS=${CERC_NA_ADDRESS} + export NITRO_VPA_ADDRESS=${CERC_VPA_ADDRESS} + export NITRO_CA_ADDRESS=${CERC_CA_ADDRESS} + else + # Read addresses from a file + # Keep retrying until found + echo "Reading Nitro addresses from ${nitro_addresses_file}" + retry_interval=5 + while true; do + if [[ -e "$nitro_addresses_file" ]]; then + export NITRO_NA_ADDRESS=$(jq -r '.nitroAdjudicatorAddress' ${nitro_addresses_file}) + export NITRO_VPA_ADDRESS=$(jq -r '.virtualPaymentAppAddress' ${nitro_addresses_file}) + export NITRO_CA_ADDRESS=$(jq -r '.consensusAppAddress' ${nitro_addresses_file}) + + break + else + echo "File not yet available, retrying in $retry_interval seconds..." + sleep $retry_interval + fi + done + fi +} + +wait_for_nitro_endpoint() { + retry_interval=5 + while true; do + curl -I -s -o /dev/null $NITRO_ENDPOINT/health + + if [ $? -eq 0 ]; then + echo "Nitro endpoint is up" + break + else + echo "Nitro endpoint not available yet, retrying in $retry_interval seconds..." + sleep $retry_interval + fi + done +} + +if [ "$NITRO_RUN_NODE_IN_PROCESS" = "true" ]; then + read_nitro_addresses +else + wait_for_nitro_endpoint +fi + +echo "Beginning the ipld-eth-server process" + +START_CMD="./ipld-eth-server" +if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then + START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec `pwd`/ipld-eth-server --continue --" +fi + +echo running: $START_CMD ${VDB_COMMAND} --config=`pwd`/config.toml +$START_CMD ${VDB_COMMAND} --config=`pwd`/config.toml +rv=$? + +if [ $rv != 0 ]; then + echo "ipld-eth-server startup failed" + exit 1 +fi diff --git a/app/data/config/keycloak/import/cerc-realm.json b/stack_orchestrator/data/config/keycloak/import/cerc-realm.json similarity index 100% rename from app/data/config/keycloak/import/cerc-realm.json rename to stack_orchestrator/data/config/keycloak/import/cerc-realm.json diff --git a/app/data/config/keycloak/keycloak.env b/stack_orchestrator/data/config/keycloak/keycloak.env similarity index 100% rename from app/data/config/keycloak/keycloak.env rename to stack_orchestrator/data/config/keycloak/keycloak.env diff --git a/app/data/config/keycloak/nginx/keycloak_proxy.conf b/stack_orchestrator/data/config/keycloak/nginx/keycloak_proxy.conf similarity index 100% rename from app/data/config/keycloak/nginx/keycloak_proxy.conf rename to stack_orchestrator/data/config/keycloak/nginx/keycloak_proxy.conf diff --git a/app/data/config/mainnet-eth-api-proxy/ethpxy.env b/stack_orchestrator/data/config/mainnet-eth-api-proxy/ethpxy.env similarity index 100% rename from app/data/config/mainnet-eth-api-proxy/ethpxy.env rename to stack_orchestrator/data/config/mainnet-eth-api-proxy/ethpxy.env diff --git a/stack_orchestrator/data/config/mainnet-eth-ipld-eth-db/db.env b/stack_orchestrator/data/config/mainnet-eth-ipld-eth-db/db.env new file mode 100644 index 00000000..4ec11109 --- /dev/null +++ b/stack_orchestrator/data/config/mainnet-eth-ipld-eth-db/db.env @@ -0,0 +1,15 @@ +DATABASE_HOSTNAME="ipld-eth-db" +DATABASE_NAME="cerc" +DATABASE_PASSWORD="CHANGEME" +DATABASE_PORT=5432 +DATABASE_USER="vdbm" + +POSTGRES_DB="${DATABASE_NAME}" +POSTGRES_PASSWORD="${DATABASE_PASSWORD}" +POSTGRES_USER="${DATABASE_USER}" + +CERC_STATEDIFF_DB_HOST="${DATABASE_HOSTNAME}" +CERC_STATEDIFF_DB_NAME="${DATABASE_NAME}" +CERC_STATEDIFF_DB_PASSWORD="${DATABASE_PASSWORD}" +CERC_STATEDIFF_DB_PORT=${DATABASE_PORT} +CERC_STATEDIFF_DB_USER="${DATABASE_USER}" diff --git a/stack_orchestrator/data/config/mainnet-eth-ipld-eth-server/config.toml b/stack_orchestrator/data/config/mainnet-eth-ipld-eth-server/config.toml new file mode 100644 index 00000000..c433df28 --- /dev/null +++ b/stack_orchestrator/data/config/mainnet-eth-ipld-eth-server/config.toml @@ -0,0 +1,33 @@ +[database] + name = "" # $DATABASE_NAME + hostname = "" # $DATABASE_HOSTNAME + port = 5432 # $DATABASE_PORT + user = "" # $DATABASE_USER + password = "" # $DATABASE_PASSWORD + +[log] + level = "info" # $LOG_LEVEL + +[server] + ipc = false + ipcPath = "" # $SERVER_IPC_PATH + ws = false + wsPath = "0.0.0.0:8080" # $SERVER_WS_PATH + http = true + httpPath = "0.0.0.0:8081" # $SERVER_HTTP_PATH + graphql = false # $SERVER_GRAPHQL + graphqlPath = "0.0.0.0:8082" # $SERVER_GRAPHQL_PATH + +[ethereum] + chainConfig = "" # ETH_CHAIN_CONFIG + chainID = "1" # $ETH_CHAIN_ID + rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP + httpPath = "mainnet-eth-geth-1:8545" # $ETH_HTTP_PATH + supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF + stateDiffTimeout = "4m" # $ETH_STATEDIFF_TIMEOUT + forwardEthCalls = false # $ETH_FORWARD_ETH_CALLS + proxyOnError = true # $ETH_PROXY_ON_ERROR + nodeID = "" # $ETH_NODE_ID + clientName = "" # $ETH_CLIENT_NAME + genesisBlock = "" # $ETH_GENESIS_BLOCK + networkID = "1" # $ETH_NETWORK_ID diff --git a/stack_orchestrator/data/config/mainnet-eth-ipld-eth-server/srv.env b/stack_orchestrator/data/config/mainnet-eth-ipld-eth-server/srv.env new file mode 100644 index 00000000..34c79ce4 --- /dev/null +++ b/stack_orchestrator/data/config/mainnet-eth-ipld-eth-server/srv.env @@ -0,0 +1,27 @@ +CERC_REMOTE_DEBUG="false" + +LOG_LEVEL="debug" + +ETH_CHAIN_ID=1 +ETH_CLIENT_NAME="Geth" +ETH_FORWARD_ETH_CALLS="false" +ETH_FORWARD_GET_STORAGE_AT="false" +ETH_GENESIS_BLOCK="0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" +ETH_HTTP_PATH="mainnet-eth-geth-1:8545" +ETH_NETWORK_ID=1 +ETH_NODE_ID=1112 +ETH_PROXY_ON_ERROR="true" +ETH_RPC_GAS_CAP=1000000000000 +ETH_SUPPORTS_STATEDIFF="true" +ETH_STATEDIFF_TIMEOUT=4m + +SERVER_HTTP_PATH=0.0.0.0:8081 +SERVER_GRAPHQL="false" +SERVER_GRAPHQLPATH=0.0.0.0:8082 + +METRICS="true" +PROM_HTTP="true" +PROM_HTTP_ADDR="0.0.0.0" +PROM_HTTP_PORT="8090" + +VDB_COMMAND="serve" diff --git a/app/data/config/mainnet-eth-keycloak/import/cerc-realm.json b/stack_orchestrator/data/config/mainnet-eth-keycloak/import/cerc-realm.json similarity index 100% rename from app/data/config/mainnet-eth-keycloak/import/cerc-realm.json rename to stack_orchestrator/data/config/mainnet-eth-keycloak/import/cerc-realm.json diff --git a/app/data/config/mainnet-eth-keycloak/keycloak.env b/stack_orchestrator/data/config/mainnet-eth-keycloak/keycloak.env similarity index 85% rename from app/data/config/mainnet-eth-keycloak/keycloak.env rename to stack_orchestrator/data/config/mainnet-eth-keycloak/keycloak.env index f37fdd30..31a19079 100644 --- a/app/data/config/mainnet-eth-keycloak/keycloak.env +++ b/stack_orchestrator/data/config/mainnet-eth-keycloak/keycloak.env @@ -1,8 +1,11 @@ POSTGRES_DB=keycloak POSTGRES_USER=keycloak POSTGRES_PASSWORD=keycloak +# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml +PGPORT=35432 KC_DB=postgres KC_DB_URL_HOST=keycloak-db +KC_DB_URL_PORT=${PGPORT} KC_DB_URL_DATABASE=${POSTGRES_DB} KC_DB_USERNAME=${POSTGRES_USER} KC_DB_PASSWORD=${POSTGRES_PASSWORD} diff --git a/app/data/config/mainnet-eth-keycloak/nginx.example b/stack_orchestrator/data/config/mainnet-eth-keycloak/nginx.example similarity index 69% rename from app/data/config/mainnet-eth-keycloak/nginx.example rename to stack_orchestrator/data/config/mainnet-eth-keycloak/nginx.example index 67095551..758f0ce1 100644 --- a/app/data/config/mainnet-eth-keycloak/nginx.example +++ b/stack_orchestrator/data/config/mainnet-eth-keycloak/nginx.example @@ -15,42 +15,49 @@ server { } upstream geth-pool { - keepalive 100; - hash $user_id consistent; - server server-a:8545; - server server-b:8545; - server server-c:8545; + server server-a:8545 max_fails=10 fail_timeout=2s; + server server-c:8545 max_fails=10 fail_timeout=2s backup; + server server-b:8545 max_fails=10 fail_timeout=2s backup; + keepalive 200; } -# self-reg happens on one server for clarity upstream reg-ui-pool { - keepalive 100; + keepalive 2; server server-a:8085; } upstream reg-api-pool { - keepalive 100; + keepalive 2; server server-a:8086; } -# auth uses server-a if available +# auth uses the reg server when available upstream auth-pool { - keepalive 100; + keepalive 10; server server-a:8080; server server-b:8080 backup; server server-c:8080 backup; } -log_format upstreamlog '[$time_local] $remote_addr $user_id - $server_name $host to: $upstream_addr: $request $status upstream_response_time $upstream_response_time msec $msec request_time $request_time'; -proxy_cache_path /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m; + +log_format upstreamlog '[$time_local] $msec $remote_addr $user_id - $server_name($host) to $upstream_addr: $request $status upstream_response_time $upstream_response_time request_time $request_time'; +proxy_cache_path /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m; + server { listen 443 ssl http2; server_name my.example.com; + keepalive_requests 500000; + keepalive_timeout 90s; + http2_max_requests 5000000; + http2_max_concurrent_streams 1024; + http2_idle_timeout 3m; + http2_recv_timeout 30s; access_log /var/log/nginx/my.example.com-access.log upstreamlog; error_log /var/log/nginx/my.example.com-error.log; ssl_certificate /etc/nginx/ssl/my.example.com/cert.pem; ssl_certificate_key /etc/nginx/ssl/my.example.com/key.pem; + ssl_session_cache shared:SSL:10m; error_page 500 502 503 504 /50x.html; location = /50x.html { @@ -60,7 +67,6 @@ server { #rewrite ^/?$ /newuser/; rewrite ^/?$ https://www.example.com/; - # geth-pool ETH API location ~ ^/v1/eth/?([^/]*)$ { set $apiKey $1; @@ -71,8 +77,8 @@ server { auth_request_set $user_id $sent_http_x_user_id; rewrite /.*$ / break; - client_max_body_size 3m; - client_body_buffer_size 3m; + client_max_body_size 3m; + client_body_buffer_size 3m; proxy_buffer_size 32k; proxy_buffers 16 32k; proxy_busy_buffers_size 96k; @@ -80,8 +86,10 @@ server { proxy_pass http://geth-pool; proxy_set_header X-Original-Remote-Addr $remote_addr; proxy_set_header X-User-Id $user_id; + proxy_http_version 1.1; + proxy_set_header Connection ""; } - + # keycloak location = /auth { internal; @@ -95,6 +103,8 @@ server { proxy_set_header X-Original-URI $request_uri; proxy_set_header X-Original-Remote-Addr $remote_addr; proxy_set_header X-Original-Host $host; + proxy_http_version 1.1; + proxy_set_header Connection ""; } location /newuser/ { diff --git a/app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py b/stack_orchestrator/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py similarity index 100% rename from app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py rename to stack_orchestrator/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py diff --git a/app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/requirements.txt b/stack_orchestrator/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/requirements.txt similarity index 100% rename from app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/requirements.txt rename to stack_orchestrator/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/requirements.txt diff --git a/app/data/config/mainnet-eth-keycloak/ui/config.yml b/stack_orchestrator/data/config/mainnet-eth-keycloak/ui/config.yml similarity index 100% rename from app/data/config/mainnet-eth-keycloak/ui/config.yml rename to stack_orchestrator/data/config/mainnet-eth-keycloak/ui/config.yml diff --git a/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json b/stack_orchestrator/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json similarity index 100% rename from app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json rename to stack_orchestrator/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json diff --git a/app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml b/stack_orchestrator/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml similarity index 100% rename from app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml rename to stack_orchestrator/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml diff --git a/app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml b/stack_orchestrator/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml similarity index 100% rename from app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml rename to stack_orchestrator/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml diff --git a/app/data/config/mainnet-eth-metrics/metrics.env b/stack_orchestrator/data/config/mainnet-eth-metrics/metrics.env similarity index 100% rename from app/data/config/mainnet-eth-metrics/metrics.env rename to stack_orchestrator/data/config/mainnet-eth-metrics/metrics.env diff --git a/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml b/stack_orchestrator/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml similarity index 100% rename from app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml rename to stack_orchestrator/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml diff --git a/stack_orchestrator/data/config/mainnet-eth-plugeth/geth.env b/stack_orchestrator/data/config/mainnet-eth-plugeth/geth.env new file mode 100644 index 00000000..5c936d36 --- /dev/null +++ b/stack_orchestrator/data/config/mainnet-eth-plugeth/geth.env @@ -0,0 +1,75 @@ +# Enable remote debugging using dlv +CERC_REMOTE_DEBUG=false + +# Enable startup script debug output. +CERC_SCRIPT_DEBUG=false + +# Simple toggle to choose either a 'full' node or an 'archive' node +# (controls the values of --syncmode --gcmode --snapshot) +CERC_GETH_MODE_QUICK_SET=archive + +# Path to plugeth plugins. +CERC_PLUGINS_DIR="/usr/local/lib/plugeth" + +# Will turn on statediffing automatically if CERC_STATEDIFF_DB_HOST exists (see ../mainnet-eth-ipld-eth-db/db.env). +CERC_RUN_STATEDIFF="detect" + +# The minimum necessary verion of the DB to enable statediffing. +CERC_STATEDIFF_DB_GOOSE_MIN_VER=18 + +# Whether all statediff-related DB statements should be logged (useful for debugging). +CERC_STATEDIFF_DB_LOG_STATEMENTS=false + +# The number of concurrent workers to process state diff objects +CERC_STATEDIFF_WORKERS=16 + +# Each statediffing node should have a unique node ID. +CERC_STATEDIFF_DB_NODE_ID=1111 + +# Optional custom node name. +# GETH_NODE_NAME="" + +# Specify any other geth CLI options. +GETH_OPTS="" + +# --cache +GETH_CACHE=1024 + +# --cache.database +GETH_CACHE_DB=50 + +# --cache.gc +GETH_CACHE_GC=25 + +# --cache.trie +GETH_CACHE_TRIE=15 + +# --datadir +GETH_DATADIR="/data" + +# --http.api +GETH_HTTP_API="eth,web3,net" + +# --authrpc.jwtsecret +GETH_JWTSECRET="/etc/mainnet-eth/jwtsecret" + +# --maxpeers +GETH_MAX_PEERS=100 + +# --rpc.evmtimeout +GETH_RPC_EVMTIMEOUT=0 + +# --rpc.gascap +GETH_RPC_GASCAP=0 + +# --txlookuplimit +GETH_TXLOOKUPLIMIT=0 + +# --verbosity +GETH_VERBOSITY=3 + +# --log.vmodule +GETH_VMODULE="rpc/*=4" + +# --ws.api +GETH_WS_API="eth,web3,net" diff --git a/app/data/config/mainnet-eth/lighthouse.env b/stack_orchestrator/data/config/mainnet-eth-plugeth/lighthouse.env similarity index 100% rename from app/data/config/mainnet-eth/lighthouse.env rename to stack_orchestrator/data/config/mainnet-eth-plugeth/lighthouse.env diff --git a/stack_orchestrator/data/config/mainnet-eth-plugeth/scripts/run-geth.sh b/stack_orchestrator/data/config/mainnet-eth-plugeth/scripts/run-geth.sh new file mode 100755 index 00000000..1971c2d0 --- /dev/null +++ b/stack_orchestrator/data/config/mainnet-eth-plugeth/scripts/run-geth.sh @@ -0,0 +1,121 @@ +#!/bin/sh +if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then + set -x +fi + +START_CMD="geth" +if [[ "true" == "$CERC_REMOTE_DEBUG" ]] && [[ -x "/usr/local/bin/dlv" ]]; then + START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --" +fi + +# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script +cleanup() { + echo "Signal received, cleaning up..." + + # Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process) + pkill -P ${geth_pid} + sleep 2 + kill $(jobs -p) + + wait + echo "Done" +} +trap 'cleanup' SIGINT SIGTERM + +MODE_FLAGS="" +if [[ "$CERC_GETH_MODE_QUICK_SET" = "archive" ]]; then + MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-full} --gcmode=${GETH_GC_MODE:-archive} --snapshot=${GETH_SNAPSHOT:-false}" +else + MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-snap} --gcmode=${GETH_GC_MODE:-full} --snapshot=${GETH_SNAPSHOT:-true}" +fi + +if [[ "${CERC_RUN_STATEDIFF}" == "detect" ]] && [[ -n "$CERC_STATEDIFF_DB_HOST" ]]; then + dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short) + dig_status_code=$? + if [[ $dig_status_code = 0 && -n $dig_result ]]; then + echo "Statediff DB at $CERC_STATEDIFF_DB_HOST" + CERC_RUN_STATEDIFF="true" + else + echo "No statediff DB available." + CERC_RUN_STATEDIFF="false" + fi +fi + +STATEDIFF_OPTS="" +if [[ "${CERC_RUN_STATEDIFF}" == "true" ]]; then + ready=0 + echo "Waiting for statediff DB..." + while [ $ready -eq 0 ]; do + sleep 1 + export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD" + result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \ + -p "$CERC_STATEDIFF_DB_PORT" \ + -U "$CERC_STATEDIFF_DB_USER" \ + -d "$CERC_STATEDIFF_DB_NAME" \ + -t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }') + if [ -n "$result" ]; then + echo "DB ready..." + if [[ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]]; then + ready=1 + else + echo "DB not at required version (want $CERC_STATEDIFF_DB_GOOSE_MIN_VER, have $result)" + fi + fi + done + + STATEDIFF_OPTS="--statediff \ + --statediff.db.host=$CERC_STATEDIFF_DB_HOST \ + --statediff.db.name=$CERC_STATEDIFF_DB_NAME \ + --statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \ + --statediff.db.password=$CERC_STATEDIFF_DB_PASSWORD \ + --statediff.db.port=$CERC_STATEDIFF_DB_PORT \ + --statediff.db.user=$CERC_STATEDIFF_DB_USER \ + --statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \ + --statediff.db.copyfrom=${CERC_STATEDIFF_DB_COPY_FROM:-true} \ + --statediff.waitforsync=${CERC_STATEDIFF_WAIT_FO_SYNC:-true} \ + --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \ + --statediff.writing=${CERC_STATEDIFF_WRITING:-true}" + + if [[ -d "${CERC_PLUGINS_DIR}" ]]; then + # With plugeth, we separate the statediff options by prefixing with ' -- ' + STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}" + fi +fi + +$START_CMD \ + $MODE_FLAGS \ + --datadir="${GETH_DATADIR}"\ + --identity="${GETH_NODE_NAME}" \ + --maxpeers=${GETH_MAX_PEERS} \ + --cache=${GETH_CACHE} \ + --cache.gc=${GETH_CACHE_GC} \ + --cache.database=${GETH_CACHE_DB} \ + --cache.trie=${GETH_CACHE_TRIE} \ + --authrpc.addr='0.0.0.0' \ + --authrpc.vhosts='*' \ + --authrpc.jwtsecret="${GETH_JWTSECRET}" \ + --http \ + --http.addr='0.0.0.0' \ + --http.api="${GETH_HTTP_API}" \ + --http.vhosts='*' \ + --metrics \ + --metrics.addr='0.0.0.0' \ + --ws \ + --ws.addr='0.0.0.0' \ + --ws.api="${GETH_WS_API}" \ + --rpc.gascap=${GETH_RPC_GASCAP} \ + --rpc.evmtimeout=${GETH_RPC_EVMTIMEOUT} \ + --txlookuplimit=${GETH_TXLOOKUPLIMIT} \ + --verbosity=${GETH_VERBOSITY} \ + --log.vmodule="${GETH_VMODULE}" \ + ${STATEDIFF_OPTS} \ + ${GETH_OPTS} & + +geth_pid=$! +wait $geth_pid + +if [[ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]]; then + while [[ 1 -eq 1 ]]; do + sleep 60 + done +fi diff --git a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh b/stack_orchestrator/data/config/mainnet-eth-plugeth/scripts/run-lighthouse.sh similarity index 100% rename from app/data/config/mainnet-eth/scripts/run-lighthouse.sh rename to stack_orchestrator/data/config/mainnet-eth-plugeth/scripts/run-lighthouse.sh diff --git a/app/data/config/mainnet-eth/geth.env b/stack_orchestrator/data/config/mainnet-eth/geth.env similarity index 99% rename from app/data/config/mainnet-eth/geth.env rename to stack_orchestrator/data/config/mainnet-eth/geth.env index a01444df..365bb5fb 100644 --- a/app/data/config/mainnet-eth/geth.env +++ b/stack_orchestrator/data/config/mainnet-eth/geth.env @@ -25,7 +25,7 @@ GETH_CACHE_GC=25 # --cache.trie GETH_CACHE_TRIE=15 -j + # --datadir GETH_DATADIR="/data" diff --git a/stack_orchestrator/data/config/mainnet-eth/lighthouse.env b/stack_orchestrator/data/config/mainnet-eth/lighthouse.env new file mode 100644 index 00000000..11fc6b69 --- /dev/null +++ b/stack_orchestrator/data/config/mainnet-eth/lighthouse.env @@ -0,0 +1,33 @@ +# Enable startup script debug output. +CERC_SCRIPT_DEBUG=false + +# Specify any other lighthouse CLI options. +LIGHTHOUSE_OPTS="" + +# Override the advertised public IP (optional) +# --enr-address +#LIGHTHOUSE_ENR_ADDRESS="" + +# --checkpoint-sync-url +LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc" + +# --checkpoint-sync-url-timeout +LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300 + +# --datadir +LIGHTHOUSE_DATADIR=/data + +# --debug-level +LIGHTHOUSE_DEBUG_LEVEL=info + +# --http-port +LIGHTHOUSE_HTTP_PORT=5052 + +# --execution-jwt +LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret + +# --metrics-port +LIGHTHOUSE_METRICS_PORT=5054 + +# --port --enr-udp-port --enr-tcp-port +LIGHTHOUSE_NETWORK_PORT=9000 diff --git a/app/data/config/mainnet-eth/scripts/run-geth.sh b/stack_orchestrator/data/config/mainnet-eth/scripts/run-geth.sh similarity index 100% rename from app/data/config/mainnet-eth/scripts/run-geth.sh rename to stack_orchestrator/data/config/mainnet-eth/scripts/run-geth.sh diff --git a/stack_orchestrator/data/config/mainnet-eth/scripts/run-lighthouse.sh b/stack_orchestrator/data/config/mainnet-eth/scripts/run-lighthouse.sh new file mode 100755 index 00000000..efda735b --- /dev/null +++ b/stack_orchestrator/data/config/mainnet-eth/scripts/run-lighthouse.sh @@ -0,0 +1,30 @@ +#!/bin/bash +if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then + set -x +fi + +ENR_OPTS="" +if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then + ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS" +fi + +exec lighthouse bn \ + --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \ + --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \ + --datadir "$LIGHTHOUSE_DATADIR" \ + --debug-level $LIGHTHOUSE_DEBUG_LEVEL \ + --disable-deposit-contract-sync \ + --disable-upnp \ + --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \ + --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \ + --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \ + --execution-jwt /etc/mainnet-eth/jwtsecret \ + --http \ + --http-address 0.0.0.0 \ + --http-port $LIGHTHOUSE_HTTP_PORT \ + --metrics \ + --metrics-address=0.0.0.0 \ + --metrics-port $LIGHTHOUSE_METRICS_PORT \ + --network mainnet \ + --port $LIGHTHOUSE_NETWORK_PORT \ + $ENR_OPTS $LIGHTHOUSE_OPTS diff --git a/app/data/config/mainnet-go-opera/go-opera.env b/stack_orchestrator/data/config/mainnet-go-opera/go-opera.env similarity index 100% rename from app/data/config/mainnet-go-opera/go-opera.env rename to stack_orchestrator/data/config/mainnet-go-opera/go-opera.env diff --git a/app/data/config/mainnet-go-opera/start-node.sh b/stack_orchestrator/data/config/mainnet-go-opera/start-node.sh similarity index 100% rename from app/data/config/mainnet-go-opera/start-node.sh rename to stack_orchestrator/data/config/mainnet-go-opera/start-node.sh diff --git a/app/data/config/fixturenet-laconicd/registry-cli-config-template.yml b/stack_orchestrator/data/config/mainnet-laconicd/registry-cli-config-template.yml similarity index 100% rename from app/data/config/fixturenet-laconicd/registry-cli-config-template.yml rename to stack_orchestrator/data/config/mainnet-laconicd/registry-cli-config-template.yml diff --git a/app/data/config/mainnet-laconicd/scripts/export-myaddress.sh b/stack_orchestrator/data/config/mainnet-laconicd/scripts/export-myaddress.sh similarity index 100% rename from app/data/config/mainnet-laconicd/scripts/export-myaddress.sh rename to stack_orchestrator/data/config/mainnet-laconicd/scripts/export-myaddress.sh diff --git a/app/data/config/mainnet-laconicd/scripts/export-mykey.sh b/stack_orchestrator/data/config/mainnet-laconicd/scripts/export-mykey.sh similarity index 100% rename from app/data/config/mainnet-laconicd/scripts/export-mykey.sh rename to stack_orchestrator/data/config/mainnet-laconicd/scripts/export-mykey.sh diff --git a/app/data/config/mainnet-laconicd/scripts/run-laconicd.sh b/stack_orchestrator/data/config/mainnet-laconicd/scripts/run-laconicd.sh similarity index 100% rename from app/data/config/mainnet-laconicd/scripts/run-laconicd.sh rename to stack_orchestrator/data/config/mainnet-laconicd/scripts/run-laconicd.sh diff --git a/app/data/config/network/wait-for-it.sh b/stack_orchestrator/data/config/network/wait-for-it.sh similarity index 100% rename from app/data/config/network/wait-for-it.sh rename to stack_orchestrator/data/config/network/wait-for-it.sh diff --git a/app/data/config/nitro-contracts/deploy.sh b/stack_orchestrator/data/config/nitro-contracts/deploy.sh similarity index 100% rename from app/data/config/nitro-contracts/deploy.sh rename to stack_orchestrator/data/config/nitro-contracts/deploy.sh diff --git a/app/data/config/optimism-contracts/hardhat-tasks/rekey-json.ts b/stack_orchestrator/data/config/optimism-contracts/hardhat-tasks/rekey-json.ts similarity index 100% rename from app/data/config/optimism-contracts/hardhat-tasks/rekey-json.ts rename to stack_orchestrator/data/config/optimism-contracts/hardhat-tasks/rekey-json.ts diff --git a/app/data/config/optimism-contracts/hardhat-tasks/send-balance.ts b/stack_orchestrator/data/config/optimism-contracts/hardhat-tasks/send-balance.ts similarity index 100% rename from app/data/config/optimism-contracts/hardhat-tasks/send-balance.ts rename to stack_orchestrator/data/config/optimism-contracts/hardhat-tasks/send-balance.ts diff --git a/app/data/config/optimism-contracts/hardhat-tasks/verify-contract-deployment.ts b/stack_orchestrator/data/config/optimism-contracts/hardhat-tasks/verify-contract-deployment.ts similarity index 100% rename from app/data/config/optimism-contracts/hardhat-tasks/verify-contract-deployment.ts rename to stack_orchestrator/data/config/optimism-contracts/hardhat-tasks/verify-contract-deployment.ts diff --git a/stack_orchestrator/data/config/ponder/base-rates-config.json b/stack_orchestrator/data/config/ponder/base-rates-config.json new file mode 100644 index 00000000..1b385875 --- /dev/null +++ b/stack_orchestrator/data/config/ponder/base-rates-config.json @@ -0,0 +1,10 @@ +{ + "freeQueriesLimit": 10, + "freeQueriesList": [], + "queries": { + "getLogEvents": "50", + "getEthLogs": "50", + "getEthBlock": "50" + }, + "mutations": {} +} diff --git a/stack_orchestrator/data/config/ponder/deploy-erc20-contract.sh b/stack_orchestrator/data/config/ponder/deploy-erc20-contract.sh new file mode 100755 index 00000000..ea8111bb --- /dev/null +++ b/stack_orchestrator/data/config/ponder/deploy-erc20-contract.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +erc20_address_file="/app/deployment/erc20-address.json" + +echo ETH_RPC_URL=${CERC_ETH_RPC_ENDPOINT} > .env +echo ACCOUNT_PRIVATE_KEY=${CERC_PRIVATE_KEY_DEPLOYER} >> .env + +# Check and keep container running if a deployment already exists (on restarts) +if [ -f ${erc20_address_file} ]; then + echo "${erc20_address_file} already exists, skipping ERC20 contract deployment" + cat ${erc20_address_file} + + # Keep the container running + tail -f +fi + +wait_for_chain_endpoint() { + # Wait till ETH RPC endpoint is available with block number > 1 + retry_interval=5 + while true; do + block_number_hex=$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' ${CERC_ETH_RPC_ENDPOINT} | jq -r '.result') + + # Check if the request call was successful + if [ $? -ne 0 ]; then + echo "RPC endpoint ${CERC_ETH_RPC_ENDPOINT} not yet available, retrying in $retry_interval seconds..." + sleep $retry_interval + continue + fi + + # Convert hex to decimal + block_number_dec=$(printf %u ${block_number_hex}) + + # Check if block number is > 1 to avoid failures in the deployment + if [ "$block_number_dec" -ge 1 ]; then + echo "RPC endpoint ${CERC_ETH_RPC_ENDPOINT} is up" + break + else + echo "RPC endpoint ${CERC_ETH_RPC_ENDPOINT} not yet available, retrying in $retry_interval seconds..." + sleep $retry_interval + continue + fi + done +} + +wait_for_chain_endpoint + +echo "Using CERC_PRIVATE_KEY_DEPLOYER from env" + +yarn token:deploy:docker --file ${erc20_address_file} + +# Keep the container running +tail -f diff --git a/app/data/config/ponder/ponder-start.sh b/stack_orchestrator/data/config/ponder/ponder-start.sh similarity index 84% rename from app/data/config/ponder/ponder-start.sh rename to stack_orchestrator/data/config/ponder/ponder-start.sh index 4fec4120..1c2bb4a2 100755 --- a/app/data/config/ponder/ponder-start.sh +++ b/stack_orchestrator/data/config/ponder/ponder-start.sh @@ -31,6 +31,21 @@ else exit 1 fi +# Read ERC20 address from a file +# Keep retrying until found +erc20_address_file="/erc20/erc20-address.json" +echo "Reading ERC20 address from ${erc20_address_file}" +retry_interval=5 +while true; do + if [[ -e "$erc20_address_file" ]]; then + ERC20_CONTRACT=$(jq -r '.address' ${erc20_address_file}) + break + else + echo "File not yet available, retrying in $retry_interval seconds..." + sleep $retry_interval + fi +done + echo "Using CERC_PONDER_NITRO_PK from env for Nitro account" echo "Using CERC_PONDER_NITRO_CHAIN_PK (account with funds) from env for sending Nitro txs" echo "Using ${CERC_PONDER_NITRO_CHAIN_URL} as the RPC endpoint for Nitro txs" @@ -56,6 +71,7 @@ echo "UPSTREAM_NITRO_PAY_AMOUNT=\"$CERC_UPSTREAM_NITRO_PAY_AMOUNT\"" >> "$env_fi echo "INDEXER_GQL_ENDPOINT=\"$CERC_INDEXER_GQL_ENDPOINT\"" >> "$env_file" echo "INDEXER_NITRO_ADDRESS=\"$CERC_INDEXER_NITRO_ADDRESS\"" >> "$env_file" echo "INDEXER_NITRO_PAY_AMOUNT=\"$CERC_INDEXER_NITRO_PAY_AMOUNT\"" >> "$env_file" +echo "ERC20_CONTRACT=\"$ERC20_CONTRACT\"" >> "$env_file" cat "$env_file" diff --git a/app/data/config/ponder/ponder.indexer.config.ts b/stack_orchestrator/data/config/ponder/ponder.indexer-1.config.ts similarity index 78% rename from app/data/config/ponder/ponder.indexer.config.ts rename to stack_orchestrator/data/config/ponder/ponder.indexer-1.config.ts index ddce62d2..31ae699c 100644 --- a/app/data/config/ponder/ponder.indexer.config.ts +++ b/stack_orchestrator/data/config/ponder/ponder.indexer-1.config.ts @@ -34,8 +34,8 @@ export const config: Config = { name: "AdventureGold", network: "fixturenet", abi: "./abis/AdventureGold.json", - address: "0x32353A6C91143bfd6C7d363B546e62a9A2489A20", - startBlock: 5, + address: process.env.ERC20_CONTRACT, + startBlock: 1, maxBlockRange: 100, }, ], @@ -49,5 +49,17 @@ export const config: Config = { contractAddresses, relayMultiAddr: process.env.RELAY_MULTIADDR!, store: "./.ponder/nitro-db", + payments: { + cache: { + maxAccounts: 1000, + accountTTLInSecs: 1800, + maxVouchersPerAccount: 1000, + voucherTTLInSecs: 300, + maxPaymentChannels: 10000, + paymentChannelTTLInSecs: 1800, + }, + ratesFile: "./base-rates-config.json", + requestTimeoutInSecs: 10, + }, }, }; diff --git a/stack_orchestrator/data/config/ponder/ponder.indexer-2.config.ts b/stack_orchestrator/data/config/ponder/ponder.indexer-2.config.ts new file mode 100644 index 00000000..c8e1b610 --- /dev/null +++ b/stack_orchestrator/data/config/ponder/ponder.indexer-2.config.ts @@ -0,0 +1,64 @@ +import { type Config, AppMode } from "@ponder/core"; + +import contractAddresses from "./nitro-addresses.json" assert { type: "json" }; + +export const config: Config = { + networks: [ + { + name: "fixturenet", + chainId: Number(process.env.PONDER_CHAIN_ID), + indexerUrl: process.env.INDEXER_GQL_ENDPOINT, + maxRpcRequestConcurrency: 1, + pollingInterval: 5000, + payments: { + nitro: { + address: process.env.INDEXER_NITRO_ADDRESS!, + fundingAmounts: { + // TODO: Pass amounts from env + directFund: "1000000000000", + virtualFund: "1000000000", + }, + }, + paidRPCMethods: [ + "eth_getLogs", + "eth_getBlockByNumber", + "eth_getBlockByHash", + ], + amount: process.env.UPSTREAM_NITRO_PAY_AMOUNT!, + }, + }, + ], + contracts: [ + { + name: "AdventureGold", + network: "fixturenet", + abi: "./abis/AdventureGold.json", + address: process.env.ERC20_CONTRACT, + startBlock: 1, + maxBlockRange: 100, + }, + ], + options: { + mode: AppMode.Indexer, + }, + nitro: { + privateKey: process.env.PONDER_NITRO_PK!, + chainPrivateKey: process.env.PONDER_NITRO_CHAIN_PK!, + chainUrl: process.env.PONDER_NITRO_CHAIN_URL!, + contractAddresses, + relayMultiAddr: process.env.RELAY_MULTIADDR!, + store: "./.ponder/nitro-db", + payments: { + cache: { + maxAccounts: 1000, + accountTTLInSecs: 1800, + maxVouchersPerAccount: 1000, + voucherTTLInSecs: 300, + maxPaymentChannels: 10000, + paymentChannelTTLInSecs: 1800, + }, + ratesFile: "./base-rates-config.json", + requestTimeoutInSecs: 10, + }, + }, +}; diff --git a/app/data/config/ponder/ponder.watcher.config.ts b/stack_orchestrator/data/config/ponder/ponder.watcher.config.ts similarity index 73% rename from app/data/config/ponder/ponder.watcher.config.ts rename to stack_orchestrator/data/config/ponder/ponder.watcher.config.ts index 72c795cc..19e4ecb1 100644 --- a/app/data/config/ponder/ponder.watcher.config.ts +++ b/stack_orchestrator/data/config/ponder/ponder.watcher.config.ts @@ -14,8 +14,8 @@ export const config: Config = { name: "AdventureGold", network: "fixturenet", abi: "./abis/AdventureGold.json", - address: "0x32353A6C91143bfd6C7d363B546e62a9A2489A20", - startBlock: 5, + address: process.env.ERC20_CONTRACT, + startBlock: 1, maxBlockRange: 100, }, ], @@ -42,5 +42,17 @@ export const config: Config = { contractAddresses, relayMultiAddr: process.env.RELAY_MULTIADDR!, store: "./.ponder/nitro-db", + payments: { + cache: { + maxAccounts: 1000, + accountTTLInSecs: 1800, + maxVouchersPerAccount: 1000, + voucherTTLInSecs: 300, + maxPaymentChannels: 10000, + paymentChannelTTLInSecs: 1800, + }, + ratesFile: "./base-rates-config.json", + requestTimeoutInSecs: 10, + }, } }; diff --git a/app/data/config/postgresql/create-pg-stat-statements.sql b/stack_orchestrator/data/config/postgresql/create-pg-stat-statements.sql similarity index 100% rename from app/data/config/postgresql/create-pg-stat-statements.sql rename to stack_orchestrator/data/config/postgresql/create-pg-stat-statements.sql diff --git a/app/data/config/postgresql/multiple-postgressql-databases.sh b/stack_orchestrator/data/config/postgresql/multiple-postgressql-databases.sh similarity index 100% rename from app/data/config/postgresql/multiple-postgressql-databases.sh rename to stack_orchestrator/data/config/postgresql/multiple-postgressql-databases.sh diff --git a/app/data/config/reth/start-lighthouse.sh b/stack_orchestrator/data/config/reth/start-lighthouse.sh similarity index 100% rename from app/data/config/reth/start-lighthouse.sh rename to stack_orchestrator/data/config/reth/start-lighthouse.sh diff --git a/app/data/config/reth/start-reth.sh b/stack_orchestrator/data/config/reth/start-reth.sh similarity index 100% rename from app/data/config/reth/start-reth.sh rename to stack_orchestrator/data/config/reth/start-reth.sh diff --git a/app/data/config/sushiswap-subgraph-v3/filecoin.js b/stack_orchestrator/data/config/sushiswap-subgraph-v3/filecoin.js similarity index 100% rename from app/data/config/sushiswap-subgraph-v3/filecoin.js rename to stack_orchestrator/data/config/sushiswap-subgraph-v3/filecoin.js diff --git a/app/data/config/sushiswap-subgraph-v3/run-blocks.sh b/stack_orchestrator/data/config/sushiswap-subgraph-v3/run-blocks.sh similarity index 100% rename from app/data/config/sushiswap-subgraph-v3/run-blocks.sh rename to stack_orchestrator/data/config/sushiswap-subgraph-v3/run-blocks.sh diff --git a/app/data/config/sushiswap-subgraph-v3/run-v3.sh b/stack_orchestrator/data/config/sushiswap-subgraph-v3/run-v3.sh similarity index 100% rename from app/data/config/sushiswap-subgraph-v3/run-v3.sh rename to stack_orchestrator/data/config/sushiswap-subgraph-v3/run-v3.sh diff --git a/app/data/config/tx-spammer/tx-spammer.env b/stack_orchestrator/data/config/tx-spammer/tx-spammer.env similarity index 100% rename from app/data/config/tx-spammer/tx-spammer.env rename to stack_orchestrator/data/config/tx-spammer/tx-spammer.env diff --git a/app/data/config/watcher-azimuth/gateway-watchers.json b/stack_orchestrator/data/config/watcher-azimuth/gateway-watchers.json similarity index 100% rename from app/data/config/watcher-azimuth/gateway-watchers.json rename to stack_orchestrator/data/config/watcher-azimuth/gateway-watchers.json diff --git a/app/data/config/watcher-azimuth/merge-toml.js b/stack_orchestrator/data/config/watcher-azimuth/merge-toml.js similarity index 100% rename from app/data/config/watcher-azimuth/merge-toml.js rename to stack_orchestrator/data/config/watcher-azimuth/merge-toml.js diff --git a/app/data/config/watcher-azimuth/start-server.sh b/stack_orchestrator/data/config/watcher-azimuth/start-server.sh similarity index 100% rename from app/data/config/watcher-azimuth/start-server.sh rename to stack_orchestrator/data/config/watcher-azimuth/start-server.sh diff --git a/app/data/config/watcher-azimuth/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml similarity index 100% rename from app/data/config/watcher-azimuth/watcher-config-template.toml rename to stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml diff --git a/app/data/config/watcher-azimuth/watcher-params.env b/stack_orchestrator/data/config/watcher-azimuth/watcher-params.env similarity index 100% rename from app/data/config/watcher-azimuth/watcher-params.env rename to stack_orchestrator/data/config/watcher-azimuth/watcher-params.env diff --git a/app/data/config/watcher-erc20/erc20-watcher.toml b/stack_orchestrator/data/config/watcher-erc20/erc20-watcher.toml similarity index 100% rename from app/data/config/watcher-erc20/erc20-watcher.toml rename to stack_orchestrator/data/config/watcher-erc20/erc20-watcher.toml diff --git a/app/data/config/watcher-erc721/erc721-watcher.toml b/stack_orchestrator/data/config/watcher-erc721/erc721-watcher.toml similarity index 100% rename from app/data/config/watcher-erc721/erc721-watcher.toml rename to stack_orchestrator/data/config/watcher-erc721/erc721-watcher.toml diff --git a/app/data/config/watcher-gelato/create-and-import-checkpoint.sh b/stack_orchestrator/data/config/watcher-gelato/create-and-import-checkpoint.sh similarity index 100% rename from app/data/config/watcher-gelato/create-and-import-checkpoint.sh rename to stack_orchestrator/data/config/watcher-gelato/create-and-import-checkpoint.sh diff --git a/app/data/config/watcher-gelato/start-job-runner.sh b/stack_orchestrator/data/config/watcher-gelato/start-job-runner.sh similarity index 100% rename from app/data/config/watcher-gelato/start-job-runner.sh rename to stack_orchestrator/data/config/watcher-gelato/start-job-runner.sh diff --git a/app/data/config/watcher-gelato/start-server.sh b/stack_orchestrator/data/config/watcher-gelato/start-server.sh similarity index 100% rename from app/data/config/watcher-gelato/start-server.sh rename to stack_orchestrator/data/config/watcher-gelato/start-server.sh diff --git a/app/data/config/watcher-gelato/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-gelato/watcher-config-template.toml similarity index 100% rename from app/data/config/watcher-gelato/watcher-config-template.toml rename to stack_orchestrator/data/config/watcher-gelato/watcher-config-template.toml diff --git a/app/data/config/watcher-gelato/watcher-params.env b/stack_orchestrator/data/config/watcher-gelato/watcher-params.env similarity index 100% rename from app/data/config/watcher-gelato/watcher-params.env rename to stack_orchestrator/data/config/watcher-gelato/watcher-params.env diff --git a/app/data/config/watcher-mobymask-v2/deploy-and-generate-invite.sh b/stack_orchestrator/data/config/watcher-mobymask-v2/deploy-and-generate-invite.sh similarity index 100% rename from app/data/config/watcher-mobymask-v2/deploy-and-generate-invite.sh rename to stack_orchestrator/data/config/watcher-mobymask-v2/deploy-and-generate-invite.sh diff --git a/app/data/config/watcher-mobymask-v2/generate-peer-ids.sh b/stack_orchestrator/data/config/watcher-mobymask-v2/generate-peer-ids.sh similarity index 100% rename from app/data/config/watcher-mobymask-v2/generate-peer-ids.sh rename to stack_orchestrator/data/config/watcher-mobymask-v2/generate-peer-ids.sh diff --git a/app/data/config/watcher-mobymask-v2/mobymask-app-config.json b/stack_orchestrator/data/config/watcher-mobymask-v2/mobymask-app-config.json similarity index 100% rename from app/data/config/watcher-mobymask-v2/mobymask-app-config.json rename to stack_orchestrator/data/config/watcher-mobymask-v2/mobymask-app-config.json diff --git a/app/data/config/watcher-mobymask-v2/mobymask-app-start.sh b/stack_orchestrator/data/config/watcher-mobymask-v2/mobymask-app-start.sh similarity index 100% rename from app/data/config/watcher-mobymask-v2/mobymask-app-start.sh rename to stack_orchestrator/data/config/watcher-mobymask-v2/mobymask-app-start.sh diff --git a/app/data/config/watcher-mobymask-v2/mobymask-params.env b/stack_orchestrator/data/config/watcher-mobymask-v2/mobymask-params.env similarity index 100% rename from app/data/config/watcher-mobymask-v2/mobymask-params.env rename to stack_orchestrator/data/config/watcher-mobymask-v2/mobymask-params.env diff --git a/app/data/config/watcher-mobymask-v2/optimism-params.env b/stack_orchestrator/data/config/watcher-mobymask-v2/optimism-params.env similarity index 100% rename from app/data/config/watcher-mobymask-v2/optimism-params.env rename to stack_orchestrator/data/config/watcher-mobymask-v2/optimism-params.env diff --git a/app/data/config/watcher-mobymask-v2/secrets-template.json b/stack_orchestrator/data/config/watcher-mobymask-v2/secrets-template.json similarity index 100% rename from app/data/config/watcher-mobymask-v2/secrets-template.json rename to stack_orchestrator/data/config/watcher-mobymask-v2/secrets-template.json diff --git a/app/data/config/watcher-mobymask-v2/set-tests-env.sh b/stack_orchestrator/data/config/watcher-mobymask-v2/set-tests-env.sh similarity index 100% rename from app/data/config/watcher-mobymask-v2/set-tests-env.sh rename to stack_orchestrator/data/config/watcher-mobymask-v2/set-tests-env.sh diff --git a/app/data/config/watcher-mobymask-v2/start-server.sh b/stack_orchestrator/data/config/watcher-mobymask-v2/start-server.sh similarity index 100% rename from app/data/config/watcher-mobymask-v2/start-server.sh rename to stack_orchestrator/data/config/watcher-mobymask-v2/start-server.sh diff --git a/app/data/config/watcher-mobymask-v2/test-app-config.json b/stack_orchestrator/data/config/watcher-mobymask-v2/test-app-config.json similarity index 100% rename from app/data/config/watcher-mobymask-v2/test-app-config.json rename to stack_orchestrator/data/config/watcher-mobymask-v2/test-app-config.json diff --git a/app/data/config/watcher-mobymask-v2/test-app-start.sh b/stack_orchestrator/data/config/watcher-mobymask-v2/test-app-start.sh similarity index 100% rename from app/data/config/watcher-mobymask-v2/test-app-start.sh rename to stack_orchestrator/data/config/watcher-mobymask-v2/test-app-start.sh diff --git a/app/data/config/watcher-mobymask-v2/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-mobymask-v2/watcher-config-template.toml similarity index 100% rename from app/data/config/watcher-mobymask-v2/watcher-config-template.toml rename to stack_orchestrator/data/config/watcher-mobymask-v2/watcher-config-template.toml diff --git a/app/data/config/watcher-mobymask-v3/deploy-and-generate-invite.sh b/stack_orchestrator/data/config/watcher-mobymask-v3/deploy-and-generate-invite.sh similarity index 100% rename from app/data/config/watcher-mobymask-v3/deploy-and-generate-invite.sh rename to stack_orchestrator/data/config/watcher-mobymask-v3/deploy-and-generate-invite.sh diff --git a/app/data/config/watcher-mobymask-v3/keys/12D3KooWAMjBkFCT9DtCnSDcxftxJzSuTBvzVojabv64cnEvX4AZ.json b/stack_orchestrator/data/config/watcher-mobymask-v3/keys/12D3KooWAMjBkFCT9DtCnSDcxftxJzSuTBvzVojabv64cnEvX4AZ.json similarity index 100% rename from app/data/config/watcher-mobymask-v3/keys/12D3KooWAMjBkFCT9DtCnSDcxftxJzSuTBvzVojabv64cnEvX4AZ.json rename to stack_orchestrator/data/config/watcher-mobymask-v3/keys/12D3KooWAMjBkFCT9DtCnSDcxftxJzSuTBvzVojabv64cnEvX4AZ.json diff --git a/app/data/config/watcher-mobymask-v3/keys/12D3KooWBNEbY3QS4y23ngupDw9PDc4bvNvRJGVRejjV9EZLjux5.json b/stack_orchestrator/data/config/watcher-mobymask-v3/keys/12D3KooWBNEbY3QS4y23ngupDw9PDc4bvNvRJGVRejjV9EZLjux5.json similarity index 100% rename from app/data/config/watcher-mobymask-v3/keys/12D3KooWBNEbY3QS4y23ngupDw9PDc4bvNvRJGVRejjV9EZLjux5.json rename to stack_orchestrator/data/config/watcher-mobymask-v3/keys/12D3KooWBNEbY3QS4y23ngupDw9PDc4bvNvRJGVRejjV9EZLjux5.json diff --git a/app/data/config/watcher-mobymask-v3/keys/12D3KooWSRH6ftgkAZsKZK7UX1Zr6Hx6YAsEepHqzopFszqfTxxi.json b/stack_orchestrator/data/config/watcher-mobymask-v3/keys/12D3KooWSRH6ftgkAZsKZK7UX1Zr6Hx6YAsEepHqzopFszqfTxxi.json similarity index 100% rename from app/data/config/watcher-mobymask-v3/keys/12D3KooWSRH6ftgkAZsKZK7UX1Zr6Hx6YAsEepHqzopFszqfTxxi.json rename to stack_orchestrator/data/config/watcher-mobymask-v3/keys/12D3KooWSRH6ftgkAZsKZK7UX1Zr6Hx6YAsEepHqzopFszqfTxxi.json diff --git a/app/data/config/watcher-mobymask-v3/mobymask-app-start.sh b/stack_orchestrator/data/config/watcher-mobymask-v3/mobymask-app-start.sh similarity index 100% rename from app/data/config/watcher-mobymask-v3/mobymask-app-start.sh rename to stack_orchestrator/data/config/watcher-mobymask-v3/mobymask-app-start.sh diff --git a/app/data/config/watcher-mobymask-v3/mobymask-params.env b/stack_orchestrator/data/config/watcher-mobymask-v3/mobymask-params.env similarity index 95% rename from app/data/config/watcher-mobymask-v3/mobymask-params.env rename to stack_orchestrator/data/config/watcher-mobymask-v3/mobymask-params.env index 1d843597..7a3f5deb 100644 --- a/app/data/config/watcher-mobymask-v3/mobymask-params.env +++ b/stack_orchestrator/data/config/watcher-mobymask-v3/mobymask-params.env @@ -4,7 +4,7 @@ DEFAULT_CERC_ETH_RPC_ENDPOINT="http://fixturenet-eth-geth-1:8545" # ETH RPC endpoint used for queries in the watcher -DEFAULT_CERC_ETH_RPC_QUERY_ENDPOINT="http://nitro-reverse-payment-proxy:8081" +DEFAULT_CERC_ETH_RPC_QUERY_ENDPOINT="http://ipld-eth-server-1:8081" # ETH RPC endpoint used for mutations in the watcher DEFAULT_CERC_ETH_RPC_MUTATION_ENDPOINT="http://fixturenet-eth-geth-1:8545" diff --git a/app/data/config/watcher-mobymask-v3/start-server.sh b/stack_orchestrator/data/config/watcher-mobymask-v3/start-server.sh similarity index 100% rename from app/data/config/watcher-mobymask-v3/start-server.sh rename to stack_orchestrator/data/config/watcher-mobymask-v3/start-server.sh diff --git a/app/data/config/watcher-mobymask-v3/watcher-config-rates.toml b/stack_orchestrator/data/config/watcher-mobymask-v3/watcher-config-rates.toml similarity index 100% rename from app/data/config/watcher-mobymask-v3/watcher-config-rates.toml rename to stack_orchestrator/data/config/watcher-mobymask-v3/watcher-config-rates.toml diff --git a/app/data/config/watcher-mobymask-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-mobymask-v3/watcher-config-template.toml similarity index 100% rename from app/data/config/watcher-mobymask-v3/watcher-config-template.toml rename to stack_orchestrator/data/config/watcher-mobymask-v3/watcher-config-template.toml diff --git a/app/data/config/watcher-mobymask/mobymask-watcher-db.sql b/stack_orchestrator/data/config/watcher-mobymask/mobymask-watcher-db.sql similarity index 100% rename from app/data/config/watcher-mobymask/mobymask-watcher-db.sql rename to stack_orchestrator/data/config/watcher-mobymask/mobymask-watcher-db.sql diff --git a/app/data/config/watcher-mobymask/mobymask-watcher.toml b/stack_orchestrator/data/config/watcher-mobymask/mobymask-watcher.toml similarity index 100% rename from app/data/config/watcher-mobymask/mobymask-watcher.toml rename to stack_orchestrator/data/config/watcher-mobymask/mobymask-watcher.toml diff --git a/app/data/config/watcher-sushiswap/erc20-watcher.toml b/stack_orchestrator/data/config/watcher-sushiswap/erc20-watcher.toml similarity index 100% rename from app/data/config/watcher-sushiswap/erc20-watcher.toml rename to stack_orchestrator/data/config/watcher-sushiswap/erc20-watcher.toml diff --git a/app/data/config/watcher-sushiswap/lotus-params.env b/stack_orchestrator/data/config/watcher-sushiswap/lotus-params.env similarity index 100% rename from app/data/config/watcher-sushiswap/lotus-params.env rename to stack_orchestrator/data/config/watcher-sushiswap/lotus-params.env diff --git a/app/data/config/watcher-sushiswap/sushi-info-watcher-test.toml b/stack_orchestrator/data/config/watcher-sushiswap/sushi-info-watcher-test.toml similarity index 100% rename from app/data/config/watcher-sushiswap/sushi-info-watcher-test.toml rename to stack_orchestrator/data/config/watcher-sushiswap/sushi-info-watcher-test.toml diff --git a/app/data/config/watcher-sushiswap/sushi-info-watcher.toml b/stack_orchestrator/data/config/watcher-sushiswap/sushi-info-watcher.toml similarity index 100% rename from app/data/config/watcher-sushiswap/sushi-info-watcher.toml rename to stack_orchestrator/data/config/watcher-sushiswap/sushi-info-watcher.toml diff --git a/app/data/config/watcher-sushiswap/sushi-watcher-test.toml b/stack_orchestrator/data/config/watcher-sushiswap/sushi-watcher-test.toml similarity index 100% rename from app/data/config/watcher-sushiswap/sushi-watcher-test.toml rename to stack_orchestrator/data/config/watcher-sushiswap/sushi-watcher-test.toml diff --git a/app/data/config/watcher-sushiswap/sushi-watcher.toml b/stack_orchestrator/data/config/watcher-sushiswap/sushi-watcher.toml similarity index 100% rename from app/data/config/watcher-sushiswap/sushi-watcher.toml rename to stack_orchestrator/data/config/watcher-sushiswap/sushi-watcher.toml diff --git a/app/data/config/watcher-uniswap-v3/erc20-watcher.toml b/stack_orchestrator/data/config/watcher-uniswap-v3/erc20-watcher.toml similarity index 100% rename from app/data/config/watcher-uniswap-v3/erc20-watcher.toml rename to stack_orchestrator/data/config/watcher-uniswap-v3/erc20-watcher.toml diff --git a/app/data/config/watcher-uniswap-v3/run.sh b/stack_orchestrator/data/config/watcher-uniswap-v3/run.sh similarity index 100% rename from app/data/config/watcher-uniswap-v3/run.sh rename to stack_orchestrator/data/config/watcher-uniswap-v3/run.sh diff --git a/app/data/config/watcher-uniswap-v3/uni-info-watcher.toml b/stack_orchestrator/data/config/watcher-uniswap-v3/uni-info-watcher.toml similarity index 100% rename from app/data/config/watcher-uniswap-v3/uni-info-watcher.toml rename to stack_orchestrator/data/config/watcher-uniswap-v3/uni-info-watcher.toml diff --git a/app/data/config/watcher-uniswap-v3/uni-watcher.toml b/stack_orchestrator/data/config/watcher-uniswap-v3/uni-watcher.toml similarity index 100% rename from app/data/config/watcher-uniswap-v3/uni-watcher.toml rename to stack_orchestrator/data/config/watcher-uniswap-v3/uni-watcher.toml diff --git a/app/data/config/watcher-uniswap-v3/watch-contract.sh b/stack_orchestrator/data/config/watcher-uniswap-v3/watch-contract.sh similarity index 100% rename from app/data/config/watcher-uniswap-v3/watch-contract.sh rename to stack_orchestrator/data/config/watcher-uniswap-v3/watch-contract.sh diff --git a/app/data/container-build/build-base.sh b/stack_orchestrator/data/container-build/build-base.sh similarity index 100% rename from app/data/container-build/build-base.sh rename to stack_orchestrator/data/container-build/build-base.sh diff --git a/app/data/container-build/cerc-act-runner-task-executor/build.sh b/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh similarity index 74% rename from app/data/container-build/cerc-act-runner-task-executor/build.sh rename to stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh index 25620a53..b625ed4b 100755 --- a/app/data/container-build/cerc-act-runner-task-executor/build.sh +++ b/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh @@ -2,4 +2,4 @@ # Build a local version of the task executor for act-runner source ${CERC_CONTAINER_BASE_DIR}/build-base.sh SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/gitea/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR} +docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR} diff --git a/app/data/container-build/cerc-act-runner/build.sh b/stack_orchestrator/data/container-build/cerc-act-runner/build.sh similarity index 100% rename from app/data/container-build/cerc-act-runner/build.sh rename to stack_orchestrator/data/container-build/cerc-act-runner/build.sh diff --git a/app/data/container-build/cerc-builder-gerbil/Dockerfile b/stack_orchestrator/data/container-build/cerc-builder-gerbil/Dockerfile similarity index 100% rename from app/data/container-build/cerc-builder-gerbil/Dockerfile rename to stack_orchestrator/data/container-build/cerc-builder-gerbil/Dockerfile diff --git a/app/data/container-build/cerc-builder-gerbil/README.md b/stack_orchestrator/data/container-build/cerc-builder-gerbil/README.md similarity index 100% rename from app/data/container-build/cerc-builder-gerbil/README.md rename to stack_orchestrator/data/container-build/cerc-builder-gerbil/README.md diff --git a/app/data/container-build/cerc-builder-gerbil/entrypoint.sh b/stack_orchestrator/data/container-build/cerc-builder-gerbil/entrypoint.sh similarity index 100% rename from app/data/container-build/cerc-builder-gerbil/entrypoint.sh rename to stack_orchestrator/data/container-build/cerc-builder-gerbil/entrypoint.sh diff --git a/app/data/container-build/cerc-builder-gerbil/install-dependencies.sh b/stack_orchestrator/data/container-build/cerc-builder-gerbil/install-dependencies.sh similarity index 100% rename from app/data/container-build/cerc-builder-gerbil/install-dependencies.sh rename to stack_orchestrator/data/container-build/cerc-builder-gerbil/install-dependencies.sh diff --git a/app/data/container-build/cerc-builder-js/Dockerfile b/stack_orchestrator/data/container-build/cerc-builder-js/Dockerfile similarity index 100% rename from app/data/container-build/cerc-builder-js/Dockerfile rename to stack_orchestrator/data/container-build/cerc-builder-js/Dockerfile diff --git a/app/data/container-build/cerc-builder-js/README.md b/stack_orchestrator/data/container-build/cerc-builder-js/README.md similarity index 100% rename from app/data/container-build/cerc-builder-js/README.md rename to stack_orchestrator/data/container-build/cerc-builder-js/README.md diff --git a/app/data/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh b/stack_orchestrator/data/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh similarity index 100% rename from app/data/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh rename to stack_orchestrator/data/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh diff --git a/app/data/container-build/cerc-builder-js/build-npm-package.sh b/stack_orchestrator/data/container-build/cerc-builder-js/build-npm-package.sh similarity index 100% rename from app/data/container-build/cerc-builder-js/build-npm-package.sh rename to stack_orchestrator/data/container-build/cerc-builder-js/build-npm-package.sh diff --git a/app/data/container-build/cerc-builder-js/check-uid.sh b/stack_orchestrator/data/container-build/cerc-builder-js/check-uid.sh similarity index 100% rename from app/data/container-build/cerc-builder-js/check-uid.sh rename to stack_orchestrator/data/container-build/cerc-builder-js/check-uid.sh diff --git a/app/data/container-build/cerc-builder-js/entrypoint.sh b/stack_orchestrator/data/container-build/cerc-builder-js/entrypoint.sh similarity index 100% rename from app/data/container-build/cerc-builder-js/entrypoint.sh rename to stack_orchestrator/data/container-build/cerc-builder-js/entrypoint.sh diff --git a/app/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh b/stack_orchestrator/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh similarity index 100% rename from app/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh rename to stack_orchestrator/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh diff --git a/app/data/container-build/cerc-eth-api-proxy/build.sh b/stack_orchestrator/data/container-build/cerc-eth-api-proxy/build.sh similarity index 100% rename from app/data/container-build/cerc-eth-api-proxy/build.sh rename to stack_orchestrator/data/container-build/cerc-eth-api-proxy/build.sh diff --git a/app/data/container-build/cerc-eth-probe/build.sh b/stack_orchestrator/data/container-build/cerc-eth-probe/build.sh similarity index 100% rename from app/data/container-build/cerc-eth-probe/build.sh rename to stack_orchestrator/data/container-build/cerc-eth-probe/build.sh diff --git a/app/data/container-build/cerc-eth-statediff-fill-service/build.sh b/stack_orchestrator/data/container-build/cerc-eth-statediff-fill-service/build.sh similarity index 100% rename from app/data/container-build/cerc-eth-statediff-fill-service/build.sh rename to stack_orchestrator/data/container-build/cerc-eth-statediff-fill-service/build.sh diff --git a/app/data/container-build/cerc-eth-statediff-service/build.sh b/stack_orchestrator/data/container-build/cerc-eth-statediff-service/build.sh similarity index 100% rename from app/data/container-build/cerc-eth-statediff-service/build.sh rename to stack_orchestrator/data/container-build/cerc-eth-statediff-service/build.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile diff --git a/app/data/container-build/cerc-fixturenet-eth-genesis/build.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/build.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-genesis/build.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/build.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-genesis/genesis/Makefile b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/Makefile similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-genesis/genesis/Makefile rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/Makefile diff --git a/app/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/import_keys.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/import_keys.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/import_keys.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/import_keys.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py diff --git a/app/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/build_el.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/build_el.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/build_el.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/build_el.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/el-config.yaml b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/el-config.yaml similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/el-config.yaml rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/el-config.yaml diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/Dockerfile similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/Dockerfile diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/build.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/build.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-geth/build.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/build.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/run-el.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/run-el.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/build.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/build.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/build.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/build.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/Makefile b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/Makefile similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/Makefile rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/Makefile diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/beacon_node.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/beacon_node.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/beacon_node.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/beacon_node.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/bootnode.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/bootnode.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/bootnode.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/bootnode.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/build_cl.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/build_cl.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/build_cl.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/build_cl.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/ready.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/ready.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/ready.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/ready.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/reset_genesis_time.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/reset_genesis_time.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/reset_genesis_time.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/reset_genesis_time.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/validator_client.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/validator_client.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/validator_client.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/validator_client.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/run-cl.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/run-cl.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/run-cl.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/run-cl.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh similarity index 94% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh index dd93b219..151cb7c1 100755 --- a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh +++ b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh @@ -20,21 +20,24 @@ if [[ $? -ne 0 ]]; then exit 1 fi -# Stop geth. -echo -n "Exporting ethdb.... " GETH_CONTAINER=`docker ps -q -f "name=${CERC_SO_COMPOSE_PROJECT}-fixturenet-eth-geth-2-1"` if [[ -z "$GETH_CONTAINER" ]]; then echo "not found" exit 1 fi +# Make sure we have the necessary tools +docker exec $GETH_CONTAINER sh -c 'apk add --no-cache jq curl' + +# Gather metadata. docker exec $GETH_CONTAINER sh -c 'rm -rf /root/tmp && mkdir -p /root/tmp/export' docker exec $GETH_CONTAINER sh -c 'ln -s /opt/testnet/build/el/geth.json /root/tmp/export/genesis.json && ln -s /root/ethdata /root/tmp/export/' docker exec $GETH_CONTAINER sh -c 'cat /root/tmp/export/genesis.json | jq ".config" > /root/tmp/export/genesis.config.json' - -# Stop geth and zip up ethdb. docker exec $GETH_CONTAINER sh -c 'curl -s --location "localhost:8545" --header "Content-Type: application/json" --data "{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"eth_getBlockByNumber\", \"params\": [\"0x0\", false]}" > /root/tmp/export/eth_getBlockByNumber_0x0.json' docker exec $GETH_CONTAINER sh -c 'curl -s --location "localhost:8545" --header "Content-Type: application/json" --data "{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"eth_blockNumber\", \"params\": []}" > /root/tmp/export/eth_blockNumber.json' + +# Stop geth and zip up ethdb. +echo -n "Exporting ethdb.... " docker exec $GETH_CONTAINER sh -c "killall geth && sleep 2 && tar chzf /root/tmp/ethdb.tgz -C /root/tmp/export ." # Copy ethdb to host. diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status-internal.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status-internal.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status-internal.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status-internal.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh diff --git a/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile b/stack_orchestrator/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile similarity index 100% rename from app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile rename to stack_orchestrator/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile diff --git a/app/data/container-build/cerc-fixturenet-plugeth-plugeth/build.sh b/stack_orchestrator/data/container-build/cerc-fixturenet-plugeth-plugeth/build.sh similarity index 100% rename from app/data/container-build/cerc-fixturenet-plugeth-plugeth/build.sh rename to stack_orchestrator/data/container-build/cerc-fixturenet-plugeth-plugeth/build.sh diff --git a/app/data/container-build/cerc-foundry/build.sh b/stack_orchestrator/data/container-build/cerc-foundry/build.sh similarity index 100% rename from app/data/container-build/cerc-foundry/build.sh rename to stack_orchestrator/data/container-build/cerc-foundry/build.sh diff --git a/app/data/container-build/cerc-go-ethereum-foundry/Dockerfile b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/Dockerfile similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/Dockerfile rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/Dockerfile diff --git a/app/data/container-build/cerc-go-ethereum-foundry/build.sh b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/build.sh similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/build.sh rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/build.sh diff --git a/app/data/container-build/cerc-go-ethereum-foundry/deploy-local-network.sh b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/deploy-local-network.sh similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/deploy-local-network.sh rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/deploy-local-network.sh diff --git a/app/data/container-build/cerc-go-ethereum-foundry/genesis-automine.json b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/genesis-automine.json similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/genesis-automine.json rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/genesis-automine.json diff --git a/app/data/container-build/cerc-go-ethereum-foundry/genesis.json b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/genesis.json similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/genesis.json rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/genesis.json diff --git a/app/data/container-build/cerc-go-ethereum-foundry/start-private-network.sh b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/start-private-network.sh similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/start-private-network.sh rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/start-private-network.sh diff --git a/app/data/container-build/cerc-go-ethereum-foundry/stateful/foundry.toml b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/foundry.toml similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/stateful/foundry.toml rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/foundry.toml diff --git a/app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/LICENSE b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/LICENSE similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/LICENSE rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/LICENSE diff --git a/app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/Makefile b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/Makefile similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/Makefile rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/Makefile diff --git a/app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/default.nix b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/default.nix similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/default.nix rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/default.nix diff --git a/app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/demo/demo.sol b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/demo/demo.sol similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/demo/demo.sol rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/demo/demo.sol diff --git a/app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/src/test.sol b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/src/test.sol similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/src/test.sol rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/lib/ds-test/src/test.sol diff --git a/app/data/container-build/cerc-go-ethereum-foundry/stateful/src/Stateful.sol b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/src/Stateful.sol similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/stateful/src/Stateful.sol rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/src/Stateful.sol diff --git a/app/data/container-build/cerc-go-ethereum-foundry/stateful/src/test/Stateful.t.sol b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/src/test/Stateful.t.sol similarity index 100% rename from app/data/container-build/cerc-go-ethereum-foundry/stateful/src/test/Stateful.t.sol rename to stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/src/test/Stateful.t.sol diff --git a/app/data/container-build/cerc-go-ethereum/build.sh b/stack_orchestrator/data/container-build/cerc-go-ethereum/build.sh similarity index 100% rename from app/data/container-build/cerc-go-ethereum/build.sh rename to stack_orchestrator/data/container-build/cerc-go-ethereum/build.sh diff --git a/stack_orchestrator/data/container-build/cerc-go-nitro/Dockerfile b/stack_orchestrator/data/container-build/cerc-go-nitro/Dockerfile new file mode 100644 index 00000000..1d0e31b4 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-go-nitro/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.21-bullseye AS builder + +# Copy files into image +WORKDIR /app +COPY . . + +# Build the binary +RUN go build -v -o nitro . + +# Reduce image size +FROM debian:bullseye-slim +RUN apt-get update +RUN apt-get install -y ca-certificates jq netcat +RUN rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=builder /app/nitro . diff --git a/stack_orchestrator/data/container-build/cerc-go-nitro/build.sh b/stack_orchestrator/data/container-build/cerc-go-nitro/build.sh new file mode 100755 index 00000000..28bdde44 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-go-nitro/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Build cerc/go-nitro + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/go-nitro:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/go-nitro diff --git a/app/data/container-build/cerc-go-opera/build.sh b/stack_orchestrator/data/container-build/cerc-go-opera/build.sh similarity index 100% rename from app/data/container-build/cerc-go-opera/build.sh rename to stack_orchestrator/data/container-build/cerc-go-opera/build.sh diff --git a/app/data/container-build/cerc-graph-node/build.sh b/stack_orchestrator/data/container-build/cerc-graph-node/build.sh similarity index 100% rename from app/data/container-build/cerc-graph-node/build.sh rename to stack_orchestrator/data/container-build/cerc-graph-node/build.sh diff --git a/app/data/container-build/cerc-ipld-eth-beacon-db/build.sh b/stack_orchestrator/data/container-build/cerc-ipld-eth-beacon-db/build.sh similarity index 100% rename from app/data/container-build/cerc-ipld-eth-beacon-db/build.sh rename to stack_orchestrator/data/container-build/cerc-ipld-eth-beacon-db/build.sh diff --git a/app/data/container-build/cerc-ipld-eth-beacon-indexer/build.sh b/stack_orchestrator/data/container-build/cerc-ipld-eth-beacon-indexer/build.sh similarity index 100% rename from app/data/container-build/cerc-ipld-eth-beacon-indexer/build.sh rename to stack_orchestrator/data/container-build/cerc-ipld-eth-beacon-indexer/build.sh diff --git a/app/data/container-build/cerc-ipld-eth-db/build.sh b/stack_orchestrator/data/container-build/cerc-ipld-eth-db/build.sh similarity index 100% rename from app/data/container-build/cerc-ipld-eth-db/build.sh rename to stack_orchestrator/data/container-build/cerc-ipld-eth-db/build.sh diff --git a/app/data/container-build/cerc-ipld-eth-server/build.sh b/stack_orchestrator/data/container-build/cerc-ipld-eth-server/build.sh similarity index 100% rename from app/data/container-build/cerc-ipld-eth-server/build.sh rename to stack_orchestrator/data/container-build/cerc-ipld-eth-server/build.sh diff --git a/app/data/container-build/cerc-keycloak-reg-api/build.sh b/stack_orchestrator/data/container-build/cerc-keycloak-reg-api/build.sh similarity index 100% rename from app/data/container-build/cerc-keycloak-reg-api/build.sh rename to stack_orchestrator/data/container-build/cerc-keycloak-reg-api/build.sh diff --git a/app/data/container-build/cerc-keycloak-reg-ui/build.sh b/stack_orchestrator/data/container-build/cerc-keycloak-reg-ui/build.sh similarity index 100% rename from app/data/container-build/cerc-keycloak-reg-ui/build.sh rename to stack_orchestrator/data/container-build/cerc-keycloak-reg-ui/build.sh diff --git a/app/data/container-build/cerc-keycloak/Dockerfile b/stack_orchestrator/data/container-build/cerc-keycloak/Dockerfile similarity index 100% rename from app/data/container-build/cerc-keycloak/Dockerfile rename to stack_orchestrator/data/container-build/cerc-keycloak/Dockerfile diff --git a/app/data/container-build/cerc-keycloak/build.sh b/stack_orchestrator/data/container-build/cerc-keycloak/build.sh similarity index 100% rename from app/data/container-build/cerc-keycloak/build.sh rename to stack_orchestrator/data/container-build/cerc-keycloak/build.sh diff --git a/app/data/container-build/cerc-laconic-console-host/Dockerfile b/stack_orchestrator/data/container-build/cerc-laconic-console-host/Dockerfile similarity index 100% rename from app/data/container-build/cerc-laconic-console-host/Dockerfile rename to stack_orchestrator/data/container-build/cerc-laconic-console-host/Dockerfile diff --git a/app/data/container-build/cerc-laconic-console-host/build.sh b/stack_orchestrator/data/container-build/cerc-laconic-console-host/build.sh similarity index 100% rename from app/data/container-build/cerc-laconic-console-host/build.sh rename to stack_orchestrator/data/container-build/cerc-laconic-console-host/build.sh diff --git a/app/data/container-build/cerc-laconic-console-host/config.yml b/stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml similarity index 100% rename from app/data/container-build/cerc-laconic-console-host/config.yml rename to stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml diff --git a/stack_orchestrator/data/container-build/cerc-laconic-dot-com/build.sh b/stack_orchestrator/data/container-build/cerc-laconic-dot-com/build.sh new file mode 100755 index 00000000..9ddcd090 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-dot-com/build.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build the laconic.com image +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh +docker build -t cerc/laconic-dot-com:local -f ${CERC_REPO_BASE_DIR}/laconic.com/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/laconic.com diff --git a/app/data/container-build/cerc-laconic-registry-cli/Dockerfile b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/Dockerfile similarity index 97% rename from app/data/container-build/cerc-laconic-registry-cli/Dockerfile rename to stack_orchestrator/data/container-build/cerc-laconic-registry-cli/Dockerfile index 7859adb4..8ff8ab7e 100644 --- a/app/data/container-build/cerc-laconic-registry-cli/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/Dockerfile @@ -54,8 +54,10 @@ RUN yarn global add @cerc-io/laconic-registry-cli # Add scripts RUN mkdir /scripts +RUN mkdir /scripts/demo-records ENV PATH="${PATH}:/scripts" COPY ./create-demo-records.sh /scripts +COPY ./demo-records /scripts/demo-records COPY ./import-key.sh /scripts COPY ./import-address.sh /scripts diff --git a/app/data/container-build/cerc-laconic-registry-cli/build.sh b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/build.sh similarity index 100% rename from app/data/container-build/cerc-laconic-registry-cli/build.sh rename to stack_orchestrator/data/container-build/cerc-laconic-registry-cli/build.sh diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh new file mode 100755 index 00000000..5a5dc34c --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Create some demo/test records in the registry +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +registry_command="laconic cns" +demo_records_dir="scripts/demo-records" + +# Check we have funds +funds_response=$(${registry_command} account get --address $(cat my-address.txt)) +funds_balance=$(echo ${funds_response} | jq -r .[0].balance[0].quantity) +echo "Balance is: ${funds_balance}" + +# Create a bond +bond_create_result=$(${registry_command} bond create --type aphoton --quantity 1000000000) +bond_id=$(echo ${bond_create_result} | jq -r .bondId) +echo "Created bond with id: ${bond_id}" + +## Publish the demo records +if [ -d $demo_records_dir ]; then + for demo_record in "${demo_records_dir}"/*; do + publish_response=$(${registry_command} record publish --filename ${demo_record} --bond-id ${bond_id}) + published_record_id=$(echo ${publish_response} | jq -r .id) + echo "Published ${demo_record} with id: ${published_record_id}" + cat ${demo_record} + done +fi diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-1.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-1.yml new file mode 100644 index 00000000..47b328cf --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-1.yml @@ -0,0 +1,7 @@ +record: + type: WebsiteRegistrationRecord + url: 'https://hello-urbit.laconic.com' + repo_registration_record_cid: QmTZQ8ZJS6mALjEM2wY71msFno6zzxFftVCiZELj9xREPx + build_artifact_cid: '~lostex-rabdur-labtul-moltev/hello-urbit' + tls_cert_cid: QmR1acEmQt7Tjmhp9cFtymie2eFcrHURQKt9kGto1TQTW1 + version: 0.2.4 diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-2.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-2.yml new file mode 100644 index 00000000..eae0cf8b --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-2.yml @@ -0,0 +1,7 @@ +record: + type: WebsiteRegistrationRecord + url: 'https://dev-hello-urbit.laconic.com' + repo_registration_record_cid: QmPbKkK7gKojnGAgKAE6bK5qBY8BcYp2YDB1e4b28NxvVv + build_artifact_cid: '~lostex-rabdur-labtul-moltev/hello-urbit-dev' + tls_cert_cid: QmWbCv4k9Q9P6XptHNJN6EupDyKVLSLRV6W6T2sAYkfBxr + version: 0.2.5 diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-3.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-3.yml new file mode 100644 index 00000000..3197b244 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-3.yml @@ -0,0 +1,7 @@ +record: + type: WebsiteRegistrationRecord + url: 'https://app.osmosis.zone' + repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D + build_artifact_cid: '~larryx-woldyr/osmosis:latest' + tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR + version: 1.3.5 diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-4.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-4.yml new file mode 100644 index 00000000..f892ac81 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-4.yml @@ -0,0 +1,7 @@ +record: + type: WebsiteRegistrationRecord + url: 'https://dev.osmosis.zone' + repo_registration_record_cid: QmPbKkK7gKojnGAgKAE6bK5qBY8BcYp2YDB1e4b28NxvVv + build_artifact_cid: '~larryx-woldyr/osmosis:dev' + tls_cert_cid: QmWbCv4k9Q9P6XptHNJN6EupDyKVLSLRV6W6T2sAYkfBxr + version: 1.3.6 diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-5.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-5.yml new file mode 100644 index 00000000..d4e21bb7 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-5.yml @@ -0,0 +1,7 @@ +record: + type: WebsiteRegistrationRecord + url: 'https://pwa-1-1-3.iglootools.xyz' + repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D + build_artifact_cid: https://ipfs.io/ipfs/QmP8jTG1m9GSDJLCbeWhVSVgEzCPPwXRdCRuJtQ5Tz9Kc9 + tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR + version: 1.1.3 diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-6.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-6.yml new file mode 100644 index 00000000..9e1ffae0 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-6.yml @@ -0,0 +1,7 @@ +record: + type: WebsiteRegistrationRecord + url: 'https://pwa-1-2-3.iglootools.xyz' + repo_registration_record_cid: QmVjYQ59z3fuKfYUV1aXhDR82JAKxdqB5JMPfUsHkJUkJq + build_artifact_cid: https://ipfs.io/ipfs/QmN2yevgjC92gzGi2BY6Q2yBWmR6JNWQ9yscoPPyy1V2Hv + tls_cert_cid: QmTAnmDMSnzqAN3yyXtFD4KpyATkeze62ZkbmXHjJ8xQW8 + version: 1.2.3 diff --git a/app/data/container-build/cerc-laconic-registry-cli/import-address.sh b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/import-address.sh similarity index 100% rename from app/data/container-build/cerc-laconic-registry-cli/import-address.sh rename to stack_orchestrator/data/container-build/cerc-laconic-registry-cli/import-address.sh diff --git a/app/data/container-build/cerc-laconic-registry-cli/import-key.sh b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/import-key.sh similarity index 100% rename from app/data/container-build/cerc-laconic-registry-cli/import-key.sh rename to stack_orchestrator/data/container-build/cerc-laconic-registry-cli/import-key.sh diff --git a/app/data/container-build/cerc-laconicd/build.sh b/stack_orchestrator/data/container-build/cerc-laconicd/build.sh similarity index 100% rename from app/data/container-build/cerc-laconicd/build.sh rename to stack_orchestrator/data/container-build/cerc-laconicd/build.sh diff --git a/app/data/container-build/cerc-lasso/build.sh b/stack_orchestrator/data/container-build/cerc-lasso/build.sh similarity index 100% rename from app/data/container-build/cerc-lasso/build.sh rename to stack_orchestrator/data/container-build/cerc-lasso/build.sh diff --git a/app/data/container-build/cerc-lighthouse-cli/build.sh b/stack_orchestrator/data/container-build/cerc-lighthouse-cli/build.sh similarity index 100% rename from app/data/container-build/cerc-lighthouse-cli/build.sh rename to stack_orchestrator/data/container-build/cerc-lighthouse-cli/build.sh diff --git a/app/data/container-build/cerc-lighthouse/Dockerfile b/stack_orchestrator/data/container-build/cerc-lighthouse/Dockerfile similarity index 100% rename from app/data/container-build/cerc-lighthouse/Dockerfile rename to stack_orchestrator/data/container-build/cerc-lighthouse/Dockerfile diff --git a/app/data/container-build/cerc-lighthouse/build.sh b/stack_orchestrator/data/container-build/cerc-lighthouse/build.sh similarity index 100% rename from app/data/container-build/cerc-lighthouse/build.sh rename to stack_orchestrator/data/container-build/cerc-lighthouse/build.sh diff --git a/app/data/container-build/cerc-lighthouse/start-lighthouse.sh b/stack_orchestrator/data/container-build/cerc-lighthouse/start-lighthouse.sh similarity index 100% rename from app/data/container-build/cerc-lighthouse/start-lighthouse.sh rename to stack_orchestrator/data/container-build/cerc-lighthouse/start-lighthouse.sh diff --git a/app/data/container-build/cerc-lotus/Dockerfile b/stack_orchestrator/data/container-build/cerc-lotus/Dockerfile similarity index 100% rename from app/data/container-build/cerc-lotus/Dockerfile rename to stack_orchestrator/data/container-build/cerc-lotus/Dockerfile diff --git a/app/data/container-build/cerc-lotus/build.sh b/stack_orchestrator/data/container-build/cerc-lotus/build.sh similarity index 100% rename from app/data/container-build/cerc-lotus/build.sh rename to stack_orchestrator/data/container-build/cerc-lotus/build.sh diff --git a/app/data/container-build/cerc-mobymask-snap/Dockerfile b/stack_orchestrator/data/container-build/cerc-mobymask-snap/Dockerfile similarity index 100% rename from app/data/container-build/cerc-mobymask-snap/Dockerfile rename to stack_orchestrator/data/container-build/cerc-mobymask-snap/Dockerfile diff --git a/app/data/container-build/cerc-mobymask-snap/build.sh b/stack_orchestrator/data/container-build/cerc-mobymask-snap/build.sh similarity index 100% rename from app/data/container-build/cerc-mobymask-snap/build.sh rename to stack_orchestrator/data/container-build/cerc-mobymask-snap/build.sh diff --git a/app/data/container-build/cerc-mobymask-ui/Dockerfile b/stack_orchestrator/data/container-build/cerc-mobymask-ui/Dockerfile similarity index 100% rename from app/data/container-build/cerc-mobymask-ui/Dockerfile rename to stack_orchestrator/data/container-build/cerc-mobymask-ui/Dockerfile diff --git a/app/data/container-build/cerc-mobymask-ui/build.sh b/stack_orchestrator/data/container-build/cerc-mobymask-ui/build.sh similarity index 100% rename from app/data/container-build/cerc-mobymask-ui/build.sh rename to stack_orchestrator/data/container-build/cerc-mobymask-ui/build.sh diff --git a/app/data/container-build/cerc-mobymask/Dockerfile b/stack_orchestrator/data/container-build/cerc-mobymask/Dockerfile similarity index 100% rename from app/data/container-build/cerc-mobymask/Dockerfile rename to stack_orchestrator/data/container-build/cerc-mobymask/Dockerfile diff --git a/app/data/container-build/cerc-mobymask/build.sh b/stack_orchestrator/data/container-build/cerc-mobymask/build.sh similarity index 100% rename from app/data/container-build/cerc-mobymask/build.sh rename to stack_orchestrator/data/container-build/cerc-mobymask/build.sh diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile new file mode 100644 index 00000000..147cec29 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -0,0 +1,55 @@ +# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile +# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster +ARG VARIANT=18-bullseye +FROM node:${VARIANT} + +ARG USERNAME=node +ARG NPM_GLOBAL=/usr/local/share/npm-global + +# Add NPM global to PATH. +ENV PATH=${NPM_GLOBAL}/bin:${PATH} +# Prevents npm from printing version warnings +ENV NPM_CONFIG_UPDATE_NOTIFIER=false + +RUN \ + # Configure global npm install location, use group to adapt to UID/GID changes + if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \ + && usermod -a -G npm ${USERNAME} \ + && umask 0002 \ + && mkdir -p ${NPM_GLOBAL} \ + && touch /usr/local/etc/npmrc \ + && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \ + && chmod g+s ${NPM_GLOBAL} \ + && npm config -g set prefix ${NPM_GLOBAL} \ + && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \ + # Install eslint + && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ + && npm cache clean --force > /dev/null 2>&1 + +# [Optional] Uncomment this section to install additional OS packages. +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends jq gettext-base + +# [Optional] Uncomment if you want to install an additional version of node using nvm +# ARG EXTRA_NODE_VERSION=10 +# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}" + +# We do this to get a yq binary from the published container, for the correct architecture we're building here +# COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq + +COPY /scripts /scripts + +# [Optional] Uncomment if you want to install more global node modules +# RUN su node -c "npm install -g " + +# RUN mkdir -p /config +# COPY ./config.yml /config + +# Install simple web server for now (use nginx perhaps later) +# RUN yarn global add http-server + +# Expose port for http +EXPOSE 3000 + +# Default command sleeps forever so docker doesn't kill it +CMD ["/scripts/start-serving-app.sh"] diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp new file mode 100644 index 00000000..f4b5d4d8 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp @@ -0,0 +1,5 @@ +FROM cerc/nextjs-base:local +WORKDIR /app +COPY . . +RUN rm -rf node_modules build .next* +RUN /scripts/build-app.sh /app diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh new file mode 100755 index 00000000..3cf5f7f4 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Build cerc/laconic-registry-cli + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +CERC_CONTAINER_BUILD_WORK_DIR=${CERC_CONTAINER_BUILD_WORK_DIR:-$SCRIPT_DIR} +CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/Dockerfile} +CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local} + +docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh new file mode 100755 index 00000000..ba1cd17d --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +WORK_DIR="${1:-./}" +SRC_DIR="${2:-.next}" +TRG_DIR="${3:-.next-r}" + +cd "${WORK_DIR}" || exit 1 + +rm -rf "$TRG_DIR" +mkdir -p "$TRG_DIR" +cp -rp "$SRC_DIR" "$TRG_DIR/" + +if [ -f ".env" ]; then + TMP_ENV=`mktemp` + declare -px > $TMP_ENV + set -a + source .env + source $TMP_ENV + set +a + rm -f $TMP_ENV +fi + +for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do + for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o '^"CERC_RUNTIME_ENV[^\"]+"$'); do + orig_name=$(echo -n "${e}" | sed 's/"//g') + cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g') + cur_val=$(echo -n "\$${cur_name}" | envsubst) + esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val") + echo "$cur_name=$cur_val" + sed -i "s/$orig_name/$esc_val/g" $f + done +done diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh new file mode 100755 index 00000000..9277abc6 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +WORK_DIR="${1:-/app}" + +cd "${WORK_DIR}" || exit 1 + +cp next.config.js next.config.dist + +npm i -g js-beautify +js-beautify next.config.dist > next.config.js + +npm install + +CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }') +MOD_EXPORTS_LINE=$(grep -n 'module.exports' next.config.js | cut -d':' -f1) + +head -$(( ${MOD_EXPORTS_LINE} - 1 )) next.config.js > next.config.js.1 + +cat > next.config.js.2 < { + a[v] = \`"CERC_RUNTIME_ENV_\${v.split(/\./).pop()}"\`; + return a; + }, {}); +} catch { + // If .env-list.json cannot be loaded, we are probably running in dev mode, so use process.env instead. + envMap = Object.keys(process.env).reduce((a, v) => { + if (v.startsWith('CERC_')) { + a[\`process.env.\${v}\`] = JSON.stringify(process.env[v]); + } + return a; + }, {}); +} +EOF + +grep 'module.exports' next.config.js > next.config.js.3 + +cat > next.config.js.4 < { + config.plugins.push(new webpack.DefinePlugin(envMap)); + return config; + }, +EOF + +tail -$(( ${CONFIG_LINES} - ${MOD_EXPORTS_LINE} + 1 )) next.config.js | grep -v 'process\.env\.' > next.config.js.5 + +cat next.config.js.* | js-beautify > next.config.js +rm next.config.js.* + +"${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json + +npm run build +rm .env-list.json \ No newline at end of file diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh new file mode 100755 index 00000000..0c0e87c9 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +WORK_DIR="${1:-./}" +TMPF=$(mktemp) + +cd "$WORK_DIR" || exit 1 + +for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' -f2); do + egrep "/$d[/$]?" .gitignore >/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + continue + fi + + for f in $(find "$d" -regex ".*.[tj]sx?$" -type f); do + cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF + done +done + +cat $TMPF | sort -u | jq --raw-input . | jq --slurp . +rm -f $TMPF diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh new file mode 100755 index 00000000..abe72935 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}" +cd "$CERC_WEBAPP_FILES_DIR" + +rm -rf .next-r +"$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r +npm start .next-r -p ${CERC_LISTEN_PORT:-3000} diff --git a/app/data/container-build/cerc-nitro-contracts/Dockerfile b/stack_orchestrator/data/container-build/cerc-nitro-contracts/Dockerfile similarity index 100% rename from app/data/container-build/cerc-nitro-contracts/Dockerfile rename to stack_orchestrator/data/container-build/cerc-nitro-contracts/Dockerfile diff --git a/app/data/container-build/cerc-nitro-contracts/build.sh b/stack_orchestrator/data/container-build/cerc-nitro-contracts/build.sh similarity index 100% rename from app/data/container-build/cerc-nitro-contracts/build.sh rename to stack_orchestrator/data/container-build/cerc-nitro-contracts/build.sh diff --git a/app/data/container-build/cerc-nitro-rpc-client/Dockerfile b/stack_orchestrator/data/container-build/cerc-nitro-rpc-client/Dockerfile similarity index 100% rename from app/data/container-build/cerc-nitro-rpc-client/Dockerfile rename to stack_orchestrator/data/container-build/cerc-nitro-rpc-client/Dockerfile diff --git a/app/data/container-build/cerc-nitro-rpc-client/build.sh b/stack_orchestrator/data/container-build/cerc-nitro-rpc-client/build.sh similarity index 100% rename from app/data/container-build/cerc-nitro-rpc-client/build.sh rename to stack_orchestrator/data/container-build/cerc-nitro-rpc-client/build.sh diff --git a/app/data/container-build/cerc-optimism-contracts/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile similarity index 100% rename from app/data/container-build/cerc-optimism-contracts/Dockerfile rename to stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile diff --git a/app/data/container-build/cerc-optimism-contracts/build.sh b/stack_orchestrator/data/container-build/cerc-optimism-contracts/build.sh similarity index 100% rename from app/data/container-build/cerc-optimism-contracts/build.sh rename to stack_orchestrator/data/container-build/cerc-optimism-contracts/build.sh diff --git a/app/data/container-build/cerc-optimism-l2geth/build.sh b/stack_orchestrator/data/container-build/cerc-optimism-l2geth/build.sh similarity index 100% rename from app/data/container-build/cerc-optimism-l2geth/build.sh rename to stack_orchestrator/data/container-build/cerc-optimism-l2geth/build.sh diff --git a/app/data/container-build/cerc-optimism-op-batcher/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile similarity index 100% rename from app/data/container-build/cerc-optimism-op-batcher/Dockerfile rename to stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile diff --git a/app/data/container-build/cerc-optimism-op-batcher/build.sh b/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/build.sh similarity index 100% rename from app/data/container-build/cerc-optimism-op-batcher/build.sh rename to stack_orchestrator/data/container-build/cerc-optimism-op-batcher/build.sh diff --git a/app/data/container-build/cerc-optimism-op-node/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile similarity index 100% rename from app/data/container-build/cerc-optimism-op-node/Dockerfile rename to stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile diff --git a/app/data/container-build/cerc-optimism-op-node/build.sh b/stack_orchestrator/data/container-build/cerc-optimism-op-node/build.sh similarity index 100% rename from app/data/container-build/cerc-optimism-op-node/build.sh rename to stack_orchestrator/data/container-build/cerc-optimism-op-node/build.sh diff --git a/app/data/container-build/cerc-optimism-op-proposer/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile similarity index 100% rename from app/data/container-build/cerc-optimism-op-proposer/Dockerfile rename to stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile diff --git a/app/data/container-build/cerc-optimism-op-proposer/build.sh b/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/build.sh similarity index 100% rename from app/data/container-build/cerc-optimism-op-proposer/build.sh rename to stack_orchestrator/data/container-build/cerc-optimism-op-proposer/build.sh diff --git a/app/data/container-build/cerc-plugeth-statediff/build.sh b/stack_orchestrator/data/container-build/cerc-plugeth-statediff/build.sh similarity index 100% rename from app/data/container-build/cerc-plugeth-statediff/build.sh rename to stack_orchestrator/data/container-build/cerc-plugeth-statediff/build.sh diff --git a/stack_orchestrator/data/container-build/cerc-plugeth-with-plugins/Dockerfile b/stack_orchestrator/data/container-build/cerc-plugeth-with-plugins/Dockerfile new file mode 100644 index 00000000..87d050ea --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-plugeth-with-plugins/Dockerfile @@ -0,0 +1,22 @@ +# Using the same golang image as used to build plugeth: https://git.vdb.to/cerc-io/plugeth/src/branch/statediff/Dockerfile +FROM golang:1.20-alpine3.18 as delve + +# Add delve so that we can do remote debugging. +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +FROM cerc/plugeth-statediff:local as statediff +FROM cerc/plugeth:local as plugeth + +FROM alpine:3.18 + +# Install tools often used in scripting, like bash, wget, and jq. +RUN apk add --no-cache ca-certificates bash wget curl python3 bind-tools postgresql-client jq + +COPY --from=delve /go/bin/dlv /usr/local/bin/ +COPY --from=plugeth /usr/local/bin/geth /usr/local/bin/ + +# Place all plugeth plugins in /usr/local/lib/plugeth +COPY --from=statediff /usr/local/lib/statediff.so /usr/local/lib/plugeth/ + +EXPOSE 8545 8546 8551 6060 30303 30303/udp 40000 +ENTRYPOINT ["geth"] diff --git a/stack_orchestrator/data/container-build/cerc-plugeth-with-plugins/build.sh b/stack_orchestrator/data/container-build/cerc-plugeth-with-plugins/build.sh new file mode 100755 index 00000000..9ab44946 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-plugeth-with-plugins/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/cerc-plugeth-with-plugins +set -x + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/plugeth-with-plugins:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR diff --git a/app/data/container-build/cerc-plugeth/build.sh b/stack_orchestrator/data/container-build/cerc-plugeth/build.sh similarity index 100% rename from app/data/container-build/cerc-plugeth/build.sh rename to stack_orchestrator/data/container-build/cerc-plugeth/build.sh diff --git a/app/data/container-build/cerc-pocket/build.sh b/stack_orchestrator/data/container-build/cerc-pocket/build.sh similarity index 100% rename from app/data/container-build/cerc-pocket/build.sh rename to stack_orchestrator/data/container-build/cerc-pocket/build.sh diff --git a/app/data/container-build/cerc-ponder/Dockerfile b/stack_orchestrator/data/container-build/cerc-ponder/Dockerfile similarity index 100% rename from app/data/container-build/cerc-ponder/Dockerfile rename to stack_orchestrator/data/container-build/cerc-ponder/Dockerfile diff --git a/app/data/container-build/cerc-ponder/build.sh b/stack_orchestrator/data/container-build/cerc-ponder/build.sh similarity index 100% rename from app/data/container-build/cerc-ponder/build.sh rename to stack_orchestrator/data/container-build/cerc-ponder/build.sh diff --git a/app/data/container-build/cerc-react-peer/Dockerfile b/stack_orchestrator/data/container-build/cerc-react-peer/Dockerfile similarity index 100% rename from app/data/container-build/cerc-react-peer/Dockerfile rename to stack_orchestrator/data/container-build/cerc-react-peer/Dockerfile diff --git a/app/data/container-build/cerc-react-peer/apply-webapp-config.sh b/stack_orchestrator/data/container-build/cerc-react-peer/apply-webapp-config.sh similarity index 100% rename from app/data/container-build/cerc-react-peer/apply-webapp-config.sh rename to stack_orchestrator/data/container-build/cerc-react-peer/apply-webapp-config.sh diff --git a/app/data/container-build/cerc-react-peer/build.sh b/stack_orchestrator/data/container-build/cerc-react-peer/build.sh similarity index 100% rename from app/data/container-build/cerc-react-peer/build.sh rename to stack_orchestrator/data/container-build/cerc-react-peer/build.sh diff --git a/app/data/container-build/cerc-react-peer/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-react-peer/start-serving-app.sh similarity index 100% rename from app/data/container-build/cerc-react-peer/start-serving-app.sh rename to stack_orchestrator/data/container-build/cerc-react-peer/start-serving-app.sh diff --git a/app/data/container-build/cerc-reth/build.sh b/stack_orchestrator/data/container-build/cerc-reth/build.sh similarity index 100% rename from app/data/container-build/cerc-reth/build.sh rename to stack_orchestrator/data/container-build/cerc-reth/build.sh diff --git a/app/data/container-build/cerc-sushiswap-subgraphs/Dockerfile b/stack_orchestrator/data/container-build/cerc-sushiswap-subgraphs/Dockerfile similarity index 100% rename from app/data/container-build/cerc-sushiswap-subgraphs/Dockerfile rename to stack_orchestrator/data/container-build/cerc-sushiswap-subgraphs/Dockerfile diff --git a/app/data/container-build/cerc-sushiswap-subgraphs/build.sh b/stack_orchestrator/data/container-build/cerc-sushiswap-subgraphs/build.sh similarity index 100% rename from app/data/container-build/cerc-sushiswap-subgraphs/build.sh rename to stack_orchestrator/data/container-build/cerc-sushiswap-subgraphs/build.sh diff --git a/app/data/container-build/cerc-sushiswap-v3-core/Dockerfile b/stack_orchestrator/data/container-build/cerc-sushiswap-v3-core/Dockerfile similarity index 100% rename from app/data/container-build/cerc-sushiswap-v3-core/Dockerfile rename to stack_orchestrator/data/container-build/cerc-sushiswap-v3-core/Dockerfile diff --git a/app/data/container-build/cerc-sushiswap-v3-core/build.sh b/stack_orchestrator/data/container-build/cerc-sushiswap-v3-core/build.sh similarity index 100% rename from app/data/container-build/cerc-sushiswap-v3-core/build.sh rename to stack_orchestrator/data/container-build/cerc-sushiswap-v3-core/build.sh diff --git a/app/data/container-build/cerc-sushiswap-v3-periphery/Dockerfile b/stack_orchestrator/data/container-build/cerc-sushiswap-v3-periphery/Dockerfile similarity index 100% rename from app/data/container-build/cerc-sushiswap-v3-periphery/Dockerfile rename to stack_orchestrator/data/container-build/cerc-sushiswap-v3-periphery/Dockerfile diff --git a/app/data/container-build/cerc-sushiswap-v3-periphery/build.sh b/stack_orchestrator/data/container-build/cerc-sushiswap-v3-periphery/build.sh similarity index 100% rename from app/data/container-build/cerc-sushiswap-v3-periphery/build.sh rename to stack_orchestrator/data/container-build/cerc-sushiswap-v3-periphery/build.sh diff --git a/app/data/container-build/cerc-test-container/Dockerfile b/stack_orchestrator/data/container-build/cerc-test-container/Dockerfile similarity index 100% rename from app/data/container-build/cerc-test-container/Dockerfile rename to stack_orchestrator/data/container-build/cerc-test-container/Dockerfile diff --git a/app/data/container-build/cerc-test-container/build.sh b/stack_orchestrator/data/container-build/cerc-test-container/build.sh similarity index 100% rename from app/data/container-build/cerc-test-container/build.sh rename to stack_orchestrator/data/container-build/cerc-test-container/build.sh diff --git a/app/data/container-build/cerc-test-container/run.sh b/stack_orchestrator/data/container-build/cerc-test-container/run.sh similarity index 100% rename from app/data/container-build/cerc-test-container/run.sh rename to stack_orchestrator/data/container-build/cerc-test-container/run.sh diff --git a/app/data/container-build/cerc-test-contract/build.sh b/stack_orchestrator/data/container-build/cerc-test-contract/build.sh similarity index 100% rename from app/data/container-build/cerc-test-contract/build.sh rename to stack_orchestrator/data/container-build/cerc-test-contract/build.sh diff --git a/app/data/container-build/cerc-tx-spammer/build.sh b/stack_orchestrator/data/container-build/cerc-tx-spammer/build.sh similarity index 100% rename from app/data/container-build/cerc-tx-spammer/build.sh rename to stack_orchestrator/data/container-build/cerc-tx-spammer/build.sh diff --git a/app/data/container-build/cerc-uniswap-v3-info/Dockerfile b/stack_orchestrator/data/container-build/cerc-uniswap-v3-info/Dockerfile similarity index 100% rename from app/data/container-build/cerc-uniswap-v3-info/Dockerfile rename to stack_orchestrator/data/container-build/cerc-uniswap-v3-info/Dockerfile diff --git a/app/data/container-build/cerc-uniswap-v3-info/build.sh b/stack_orchestrator/data/container-build/cerc-uniswap-v3-info/build.sh similarity index 100% rename from app/data/container-build/cerc-uniswap-v3-info/build.sh rename to stack_orchestrator/data/container-build/cerc-uniswap-v3-info/build.sh diff --git a/app/data/container-build/cerc-watcher-azimuth/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-azimuth/Dockerfile similarity index 100% rename from app/data/container-build/cerc-watcher-azimuth/Dockerfile rename to stack_orchestrator/data/container-build/cerc-watcher-azimuth/Dockerfile diff --git a/app/data/container-build/cerc-watcher-azimuth/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-azimuth/build.sh similarity index 100% rename from app/data/container-build/cerc-watcher-azimuth/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-azimuth/build.sh diff --git a/stack_orchestrator/data/container-build/cerc-watcher-erc20/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-erc20/Dockerfile new file mode 100644 index 00000000..19be25ee --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-erc20/Dockerfile @@ -0,0 +1,10 @@ +FROM node:16.17.1-alpine3.16 + +RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq + +WORKDIR /app + +COPY . . + +RUN echo "Building erc20-watcher-ts" && \ + yarn && yarn build diff --git a/app/data/container-build/cerc-watcher-erc20/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-erc20/build.sh similarity index 81% rename from app/data/container-build/cerc-watcher-erc20/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-erc20/build.sh index 67ac0d24..6817968b 100755 --- a/app/data/container-build/cerc-watcher-erc20/build.sh +++ b/stack_orchestrator/data/container-build/cerc-watcher-erc20/build.sh @@ -6,4 +6,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh # See: https://stackoverflow.com/a/246128/1701505 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/watcher-erc20:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/watcher-ts +docker build -t cerc/watcher-erc20:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/erc20-watcher-ts diff --git a/app/data/container-build/cerc-watcher-erc721/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-erc721/Dockerfile similarity index 100% rename from app/data/container-build/cerc-watcher-erc721/Dockerfile rename to stack_orchestrator/data/container-build/cerc-watcher-erc721/Dockerfile diff --git a/app/data/container-build/cerc-watcher-erc721/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-erc721/build.sh similarity index 100% rename from app/data/container-build/cerc-watcher-erc721/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-erc721/build.sh diff --git a/app/data/container-build/cerc-watcher-gelato/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-gelato/Dockerfile similarity index 100% rename from app/data/container-build/cerc-watcher-gelato/Dockerfile rename to stack_orchestrator/data/container-build/cerc-watcher-gelato/Dockerfile diff --git a/app/data/container-build/cerc-watcher-gelato/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-gelato/build.sh similarity index 100% rename from app/data/container-build/cerc-watcher-gelato/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-gelato/build.sh diff --git a/app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-mobymask-v2/Dockerfile similarity index 100% rename from app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile rename to stack_orchestrator/data/container-build/cerc-watcher-mobymask-v2/Dockerfile diff --git a/app/data/container-build/cerc-watcher-mobymask-v2/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-mobymask-v2/build.sh similarity index 100% rename from app/data/container-build/cerc-watcher-mobymask-v2/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-mobymask-v2/build.sh diff --git a/app/data/container-build/cerc-watcher-mobymask-v3/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-mobymask-v3/Dockerfile similarity index 100% rename from app/data/container-build/cerc-watcher-mobymask-v3/Dockerfile rename to stack_orchestrator/data/container-build/cerc-watcher-mobymask-v3/Dockerfile diff --git a/app/data/container-build/cerc-watcher-mobymask-v3/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-mobymask-v3/build.sh similarity index 100% rename from app/data/container-build/cerc-watcher-mobymask-v3/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-mobymask-v3/build.sh diff --git a/app/data/container-build/cerc-watcher-mobymask/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-mobymask/Dockerfile similarity index 100% rename from app/data/container-build/cerc-watcher-mobymask/Dockerfile rename to stack_orchestrator/data/container-build/cerc-watcher-mobymask/Dockerfile diff --git a/app/data/container-build/cerc-watcher-mobymask/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-mobymask/build.sh similarity index 100% rename from app/data/container-build/cerc-watcher-mobymask/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-mobymask/build.sh diff --git a/app/data/container-build/cerc-watcher-sushiswap/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap/Dockerfile similarity index 100% rename from app/data/container-build/cerc-watcher-sushiswap/Dockerfile rename to stack_orchestrator/data/container-build/cerc-watcher-sushiswap/Dockerfile diff --git a/app/data/container-build/cerc-watcher-sushiswap/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap/build.sh similarity index 100% rename from app/data/container-build/cerc-watcher-sushiswap/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-sushiswap/build.sh diff --git a/app/data/container-build/cerc-watcher-ts/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-ts/Dockerfile similarity index 83% rename from app/data/container-build/cerc-watcher-ts/Dockerfile rename to stack_orchestrator/data/container-build/cerc-watcher-ts/Dockerfile index ec6c00f8..1c6f06a1 100644 --- a/app/data/container-build/cerc-watcher-ts/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-watcher-ts/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18.17.1-alpine3.18 +FROM node:18.17.1-alpine3.17 RUN apk --update --no-cache add git python3 alpine-sdk jq diff --git a/app/data/container-build/cerc-watcher-ts/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-ts/build.sh similarity index 100% rename from app/data/container-build/cerc-watcher-ts/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-ts/build.sh diff --git a/app/data/container-build/cerc-watcher-uniswap-v3/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-uniswap-v3/Dockerfile similarity index 100% rename from app/data/container-build/cerc-watcher-uniswap-v3/Dockerfile rename to stack_orchestrator/data/container-build/cerc-watcher-uniswap-v3/Dockerfile diff --git a/app/data/container-build/cerc-watcher-uniswap-v3/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-uniswap-v3/build.sh similarity index 100% rename from app/data/container-build/cerc-watcher-uniswap-v3/build.sh rename to stack_orchestrator/data/container-build/cerc-watcher-uniswap-v3/build.sh diff --git a/app/data/container-build/cerc-webapp-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile similarity index 100% rename from app/data/container-build/cerc-webapp-base/Dockerfile rename to stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile diff --git a/app/data/container-build/cerc-webapp-base/apply-webapp-config.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/apply-webapp-config.sh similarity index 100% rename from app/data/container-build/cerc-webapp-base/apply-webapp-config.sh rename to stack_orchestrator/data/container-build/cerc-webapp-base/apply-webapp-config.sh diff --git a/app/data/container-build/cerc-webapp-base/build.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/build.sh similarity index 100% rename from app/data/container-build/cerc-webapp-base/build.sh rename to stack_orchestrator/data/container-build/cerc-webapp-base/build.sh diff --git a/app/data/container-build/cerc-webapp-base/config.yml b/stack_orchestrator/data/container-build/cerc-webapp-base/config.yml similarity index 100% rename from app/data/container-build/cerc-webapp-base/config.yml rename to stack_orchestrator/data/container-build/cerc-webapp-base/config.yml diff --git a/app/data/container-build/cerc-webapp-base/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh similarity index 100% rename from app/data/container-build/cerc-webapp-base/start-serving-app.sh rename to stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh diff --git a/app/data/container-build/default-build.sh b/stack_orchestrator/data/container-build/default-build.sh similarity index 100% rename from app/data/container-build/default-build.sh rename to stack_orchestrator/data/container-build/default-build.sh diff --git a/app/data/container-image-list.txt b/stack_orchestrator/data/container-image-list.txt similarity index 100% rename from app/data/container-image-list.txt rename to stack_orchestrator/data/container-image-list.txt diff --git a/app/data/npm-package-list.txt b/stack_orchestrator/data/npm-package-list.txt similarity index 100% rename from app/data/npm-package-list.txt rename to stack_orchestrator/data/npm-package-list.txt diff --git a/app/data/pod-list.txt b/stack_orchestrator/data/pod-list.txt similarity index 93% rename from app/data/pod-list.txt rename to stack_orchestrator/data/pod-list.txt index ed7f17d5..4ba1bac0 100644 --- a/app/data/pod-list.txt +++ b/stack_orchestrator/data/pod-list.txt @@ -38,7 +38,8 @@ fixturenet-sushiswap-subgraph-v3 watcher-mobymask-v3 mobymask-app-v3 go-nitro +nitro-rpc-client nitro-contracts -nitro-reverse-payment-proxy mobymask-snap ponder +ipld-eth-server-payments diff --git a/app/data/repository-list.txt b/stack_orchestrator/data/repository-list.txt similarity index 100% rename from app/data/repository-list.txt rename to stack_orchestrator/data/repository-list.txt diff --git a/stack_orchestrator/data/stacks/act-runner/README.md b/stack_orchestrator/data/stacks/act-runner/README.md new file mode 100644 index 00000000..3c6dd7b1 --- /dev/null +++ b/stack_orchestrator/data/stacks/act-runner/README.md @@ -0,0 +1,13 @@ +# act-runner stack + +## Example + +``` +$ laconic-so --stack act-runner deploy init --output act-runner-1.yml --config CERC_GITEA_RUNNER_REGISTRATION_TOKEN=FOO +$ laconic-so --stack act-runner deploy create --spec-file act-runner-1.yml --deployment-dir ~/opt/deployments/act-runner-1 +$ laconic-so deployment --dir ~/opt/deployments/act-runner-1 up + +$ laconic-so --stack act-runner deploy init --output act-runner-2.yml --config CERC_GITEA_RUNNER_REGISTRATION_TOKEN=BAR +$ laconic-so --stack act-runner deploy create --spec-file act-runner-2.yml --deployment-dir ~/opt/deployments/act-runner-2 +$ laconic-so deployment --dir ~/opt/deployments/act-runner-2 up +``` diff --git a/stack_orchestrator/data/stacks/act-runner/stack.yml b/stack_orchestrator/data/stacks/act-runner/stack.yml new file mode 100644 index 00000000..a236fccf --- /dev/null +++ b/stack_orchestrator/data/stacks/act-runner/stack.yml @@ -0,0 +1,15 @@ +version: "1.1" +name: act-runner +description: "Local act-runner" +repos: + - git.vdb.to/cerc-io/hosting + - gitea.com/gitea/act_runner +containers: + - cerc/act-runner + - cerc/act-runner-task-executor +pods: + - name: act-runner + repository: cerc-io/hosting + path: act-runner + pre_start_command: "pre_start.sh" + post_start_command: "post_start.sh" diff --git a/app/data/stacks/azimuth/README.md b/stack_orchestrator/data/stacks/azimuth/README.md similarity index 100% rename from app/data/stacks/azimuth/README.md rename to stack_orchestrator/data/stacks/azimuth/README.md diff --git a/app/data/stacks/azimuth/stack.yml b/stack_orchestrator/data/stacks/azimuth/stack.yml similarity index 100% rename from app/data/stacks/azimuth/stack.yml rename to stack_orchestrator/data/stacks/azimuth/stack.yml diff --git a/app/data/stacks/build-support/README.md b/stack_orchestrator/data/stacks/build-support/README.md similarity index 100% rename from app/data/stacks/build-support/README.md rename to stack_orchestrator/data/stacks/build-support/README.md diff --git a/app/data/stacks/build-support/stack.yml b/stack_orchestrator/data/stacks/build-support/stack.yml similarity index 100% rename from app/data/stacks/build-support/stack.yml rename to stack_orchestrator/data/stacks/build-support/stack.yml diff --git a/stack_orchestrator/data/stacks/chain-chunker/README.md b/stack_orchestrator/data/stacks/chain-chunker/README.md new file mode 100644 index 00000000..e69de29b diff --git a/app/data/stacks/chain-chunker/stack.yml b/stack_orchestrator/data/stacks/chain-chunker/stack.yml similarity index 87% rename from app/data/stacks/chain-chunker/stack.yml rename to stack_orchestrator/data/stacks/chain-chunker/stack.yml index d85aa057..2705f69a 100644 --- a/app/data/stacks/chain-chunker/stack.yml +++ b/stack_orchestrator/data/stacks/chain-chunker/stack.yml @@ -6,8 +6,10 @@ repos: - git.vdb.to/cerc-io/eth-statediff-service@v5 - git.vdb.to/cerc-io/ipld-eth-db@v5 - git.vdb.to/cerc-io/ipld-eth-server@v5 + - git.vdb.to/cerc-io/plugeth@statediff containers: - cerc/ipld-eth-state-snapshot - cerc/eth-statediff-service - cerc/ipld-eth-db - cerc/ipld-eth-server + - cerc/plugeth diff --git a/app/data/stacks/erc20/README.md b/stack_orchestrator/data/stacks/erc20/README.md similarity index 100% rename from app/data/stacks/erc20/README.md rename to stack_orchestrator/data/stacks/erc20/README.md diff --git a/app/data/stacks/erc20/stack.yml b/stack_orchestrator/data/stacks/erc20/stack.yml similarity index 92% rename from app/data/stacks/erc20/stack.yml rename to stack_orchestrator/data/stacks/erc20/stack.yml index 48b8f6fb..70bbc365 100644 --- a/app/data/stacks/erc20/stack.yml +++ b/stack_orchestrator/data/stacks/erc20/stack.yml @@ -4,8 +4,8 @@ repos: - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5 - git.vdb.to/cerc-io/ipld-eth-db@v5 - git.vdb.to/cerc-io/ipld-eth-server@v1.11.6-statediff-v5 - - github.com/cerc-io/watcher-ts - github.com/dboreham/foundry + - github.com/cerc-io/erc20-watcher-ts containers: - cerc/foundry - cerc/go-ethereum diff --git a/app/data/stacks/erc721/README.md b/stack_orchestrator/data/stacks/erc721/README.md similarity index 100% rename from app/data/stacks/erc721/README.md rename to stack_orchestrator/data/stacks/erc721/README.md diff --git a/app/data/stacks/erc721/stack.yml b/stack_orchestrator/data/stacks/erc721/stack.yml similarity index 100% rename from app/data/stacks/erc721/stack.yml rename to stack_orchestrator/data/stacks/erc721/stack.yml diff --git a/app/data/stacks/fixturenet-eth-loaded/README.md b/stack_orchestrator/data/stacks/fixturenet-eth-loaded/README.md similarity index 100% rename from app/data/stacks/fixturenet-eth-loaded/README.md rename to stack_orchestrator/data/stacks/fixturenet-eth-loaded/README.md diff --git a/app/data/stacks/fixturenet-eth-loaded/stack.yml b/stack_orchestrator/data/stacks/fixturenet-eth-loaded/stack.yml similarity index 100% rename from app/data/stacks/fixturenet-eth-loaded/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-eth-loaded/stack.yml diff --git a/app/data/stacks/fixturenet-eth-tx/README.md b/stack_orchestrator/data/stacks/fixturenet-eth-tx/README.md similarity index 100% rename from app/data/stacks/fixturenet-eth-tx/README.md rename to stack_orchestrator/data/stacks/fixturenet-eth-tx/README.md diff --git a/app/data/stacks/fixturenet-eth-tx/stack.yml b/stack_orchestrator/data/stacks/fixturenet-eth-tx/stack.yml similarity index 100% rename from app/data/stacks/fixturenet-eth-tx/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-eth-tx/stack.yml diff --git a/app/data/stacks/fixturenet-eth/README.md b/stack_orchestrator/data/stacks/fixturenet-eth/README.md similarity index 100% rename from app/data/stacks/fixturenet-eth/README.md rename to stack_orchestrator/data/stacks/fixturenet-eth/README.md diff --git a/app/data/stacks/fixturenet-eth/stack.yml b/stack_orchestrator/data/stacks/fixturenet-eth/stack.yml similarity index 100% rename from app/data/stacks/fixturenet-eth/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-eth/stack.yml diff --git a/app/data/stacks/fixturenet-laconic-loaded/README.md b/stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md similarity index 100% rename from app/data/stacks/fixturenet-laconic-loaded/README.md rename to stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md diff --git a/app/data/stacks/fixturenet-laconic-loaded/stack.yml b/stack_orchestrator/data/stacks/fixturenet-laconic-loaded/stack.yml similarity index 79% rename from app/data/stacks/fixturenet-laconic-loaded/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-laconic-loaded/stack.yml index 523a7091..6c8aae4a 100644 --- a/app/data/stacks/fixturenet-laconic-loaded/stack.yml +++ b/stack_orchestrator/data/stacks/fixturenet-laconic-loaded/stack.yml @@ -2,14 +2,14 @@ version: "1.1" name: fixturenet-laconic-loaded description: "A full featured laconic fixturenet" repos: - - github.com/cerc-io/laconicd + - git.vdb.to/cerc-io/laconicd - github.com/lirewine/debug - github.com/lirewine/crypto - github.com/lirewine/gem - github.com/lirewine/sdk - - github.com/cerc-io/laconic-sdk - - github.com/cerc-io/laconic-registry-cli - - github.com/cerc-io/laconic-console + - git.vdb.to/cerc-io/laconic-sdk + - git.vdb.to/cerc-io/laconic-registry-cli + - git.vdb.to/cerc-io/laconic-console npms: - laconic-sdk - laconic-registry-cli diff --git a/app/data/stacks/fixturenet-laconicd/README.md b/stack_orchestrator/data/stacks/fixturenet-laconicd/README.md similarity index 100% rename from app/data/stacks/fixturenet-laconicd/README.md rename to stack_orchestrator/data/stacks/fixturenet-laconicd/README.md diff --git a/app/data/stacks/fixturenet-laconicd/stack.yml b/stack_orchestrator/data/stacks/fixturenet-laconicd/stack.yml similarity index 72% rename from app/data/stacks/fixturenet-laconicd/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-laconicd/stack.yml index ce0b1946..adab9e3e 100644 --- a/app/data/stacks/fixturenet-laconicd/stack.yml +++ b/stack_orchestrator/data/stacks/fixturenet-laconicd/stack.yml @@ -2,9 +2,9 @@ version: "1.0" name: fixturenet-laconicd description: "A laconicd fixturenet" repos: - - github.com/cerc-io/laconicd - - github.com/cerc-io/laconic-sdk - - github.com/cerc-io/laconic-registry-cli + - git.vdb.to/cerc-io/laconicd + - git.vdb.to/cerc-io/laconic-sdk + - git.vdb.to/cerc-io/laconic-registry-cli npms: - laconic-sdk - laconic-registry-cli diff --git a/app/data/stacks/fixturenet-lotus/README.md b/stack_orchestrator/data/stacks/fixturenet-lotus/README.md similarity index 100% rename from app/data/stacks/fixturenet-lotus/README.md rename to stack_orchestrator/data/stacks/fixturenet-lotus/README.md diff --git a/app/data/stacks/fixturenet-lotus/stack.yml b/stack_orchestrator/data/stacks/fixturenet-lotus/stack.yml similarity index 100% rename from app/data/stacks/fixturenet-lotus/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-lotus/stack.yml diff --git a/app/data/stacks/fixturenet-optimism/README.md b/stack_orchestrator/data/stacks/fixturenet-optimism/README.md similarity index 100% rename from app/data/stacks/fixturenet-optimism/README.md rename to stack_orchestrator/data/stacks/fixturenet-optimism/README.md diff --git a/app/data/stacks/fixturenet-optimism/l2-only.md b/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md similarity index 100% rename from app/data/stacks/fixturenet-optimism/l2-only.md rename to stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md diff --git a/app/data/stacks/fixturenet-optimism/stack.yml b/stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml similarity index 100% rename from app/data/stacks/fixturenet-optimism/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml diff --git a/stack_orchestrator/data/stacks/fixturenet-payments/README.md b/stack_orchestrator/data/stacks/fixturenet-payments/README.md new file mode 100644 index 00000000..ebde79ee --- /dev/null +++ b/stack_orchestrator/data/stacks/fixturenet-payments/README.md @@ -0,0 +1,84 @@ +# fixturenet-payments + +Instructions to setup and deploy an end-to-end fixturenet-payments stack, on a local machine. Some tips are included for running on a remote cloud machine. + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack fixturenet-payments setup-repositories --pull +``` + +Build the container images: + +```bash +laconic-so --stack fixturenet-payments build-containers +``` + +## Deploy + +Deploy the stack: + +```bash +laconic-so --stack fixturenet-payments deploy --cluster payments up +``` + +```bash +# Exposed on host ports: +# 32***: geth in statediffing mode and ipld-eth-server(s) +# 4005: in-process go-nitro node's RPC endpoint +# 3005: in-process go-nitro node's p2p TCP endpoint +# 5005: in-process go-nitro node's p2p WS endpoint +# 4006: out-of-process go-nitro node's RPC endpoint +# 3006: out-of-process go-nitro node's p2p TCP endpoint +# 5006: out-of-process go-nitro node's p2p WS endpoint +# 15432: MobyMask v3 watcher's db endpoint +# 3001: MobyMask v3 watcher endpoint +# 9090: MobyMask v3 watcher relay node endpoint +# 8080: MobyMask snap +# 3004: MobyMask v3 app +# 42070: Ponder indexer-1 +# 42071: Ponder indexer-2 +# 42069: Ponder watcher +``` + +If running in the cloud, ensure all the of the above ports are open. The geth port can be retrieved with: + +```bash +docker port payments-fixturenet-eth-geth-1-1 8545 +``` + +Then for every port above, run each line in a new terminal window (or use `screen`): + +```bash +ssh -L 4005:localhost:4005 user@ +ssh -L 5005:localhost:5005 user@ +ssh -L 8081:localhost:8081 user@ +# ... and so on for every port +``` + +This will allow you to access the entirety of the app as if it were running locally. + +## Demo + +- Follow the [mobymask-demo](./mobymask-demo.md) to try out MobyMask end-to-end payments. +- Follow the [ponder-demo](./ponder-demo.md) to try out Ponder end-to-end payments. + +## Clean up + +Stop all the services running in background: + +```bash +laconic-so --stack fixturenet-payments deploy --cluster payments down 30 +``` + +Clear volumes created by this stack: + +```bash +# List all relevant volumes +docker volume ls -q --filter "name=payments" + +# Remove all the listed volumes +docker volume rm $(docker volume ls -q --filter "name=payments") +``` diff --git a/stack_orchestrator/data/stacks/fixturenet-payments/mobymask-demo.md b/stack_orchestrator/data/stacks/fixturenet-payments/mobymask-demo.md new file mode 100644 index 00000000..65d93e85 --- /dev/null +++ b/stack_orchestrator/data/stacks/fixturenet-payments/mobymask-demo.md @@ -0,0 +1,223 @@ +# Demo + +Stack components: +* `ipld-eth-db` database for statediffed data +* Local geth + lighthouse blockchain "fixturenet" running in statediffing mode +* `ipld-eth-server-1` which runs an ETH RPC API and a GQL server; they both serve data from `ipld-eth-db` + * It runs an in-process go-nitro node for payments required for configured RPC requests +* A MobyMask v3 watcher that pays the `ipld-eth-server-1` for ETH RPC requests +* A MobyMask v3 app that pays the watcher for reads (GQL queries) and writes + +## Setup + +* On starting the stack, MobyMask watcher creates a payment channel with the `ipld-eth-server-1`'s (in-process) Nitro node. Check watcher logs and wait for the same: + + ```bash + docker logs -f $(docker ps -aq --filter name="mobymask-watcher-server") + + # Expected output: + # vulcanize:server Peer ID: 12D3KooWKLqLWU82VU7jmsmQMruRvZWhoBoVsf1UHchM5Nuq9ymY + # vulcanize:server Using chain URL http://fixturenet-eth-geth-1:8546 for Nitro node + # ... + # ts-nitro:util:nitro Ledger channel created with id 0x65703ccdfacab09ac35367bdbe6c5a337e7a6651aad526807607b1c59b28bc1e + # ... + # ts-nitro:util:nitro Virtual payment channel created with id 0x29ff1335d73391a50e8fde3e9b34f00c3d81c39ddc7f89187f44dd51df96140e + # vulcanize:server Starting server... +0ms + ``` + +* Keep the above command running to keep track of incoming payments and GQL requests from the MobyMask app + +* In another terminal, export the payment channel id to a variable: + + ```bash + export WATCHER_UPSTREAM_PAYMENT_CHANNEL= + ``` + +* Check the payment channel status: + + ```bash + docker exec payments-nitro-rpc-client-1 npm exec -c "nitro-rpc-client get-payment-channel $WATCHER_UPSTREAM_PAYMENT_CHANNEL -s false -h ipld-eth-server-1 -p 4005" + + # Expected output: + # { + # ID: '0x8c0d17639bd2ba07dbcd248304a8f3c6c7276bfe25c2b87fe41f461e20f33f01', + # Status: 'Open', + # Balance: { + # AssetAddress: '0x0000000000000000000000000000000000000000', + # Payee: '0xaaa6628ec44a8a742987ef3a114ddfe2d4f7adce', + # Payer: '0xbbb676f9cff8d242e9eac39d063848807d3d1d94', + # PaidSoFar: 0n, + # RemainingFunds: 1000000000n + # } + # } + ``` + +* In another terminal, check `ipld-eth-server-1`'s logs to keep track of incoming payments and RPC requests from the MobyMask watcher: + + ```bash + docker logs -f $(docker ps -aq --filter name="ipld-eth-server-1") + ``` + +* MetaMask flask wallet setup for running the MobyMask app: + + * Get the geth node’s port mapped to host: + + ```bash + docker port payments-fixturenet-eth-geth-1-1 8545 + ``` + + * In MetaMask, add a custom network with the following settings: + + ```bash + # Network name + Local fixturenet + + # New RPC URL + http://127.0.0.1: + + # Chain ID + 1212 + + # Currency symbol + ETH + ``` + + * Import a faucet account with the following private key: + + ```bash + # Faucet PK + # 0x570b909da9669b2f35a0b1ac70b8358516d55ae1b5b3710e95e9a94395090597 + ``` + + * Create an additional account for usage in the app; fund it from the faucet account + +* Get the generated root invite link for MobyMask from contract deployment container logs: + + ```bash + docker logs -f $(docker ps -aq --filter name="mobymask-1") + + # Expected output: + # ... + # "key": "0x60e706fda4639fe0a8eb102cb0ce81231cf6e819f41cb4eadf72d865ea4c11ad" + # } + # http://127.0.0.1:3004/#/members?invitation= + ``` + +## Run + +### MobyMask App + +* Open app in a browser (where MetaMask was setup) using the invite link + +* Run the following in browser console to enable logs: + + ```bash + localStorage.debug = 'ts-nitro:*' + # Refresh the tab for taking effect + ``` + +* In the app’s debug panel, check that the peer gets connected to relay node and watcher peer + +* Open the `NITRO` tab in debug panel + * Click on `Connect Wallet` to connect to MetaMask (make sure that the newly funded account is active) + * Click on `Connect Snap` to install/connect snap + +* Perform `DIRECT FUND` with the preset amount and wait for the MetaMask confirmation prompt to appear; confirm the transaction and wait for a ledger channel to be created with the watcher + +* Perform `VIRTUAL FUND` with amount set to `10000` and wait for a payment channel to be created with the watcher + +* Perform phisher status check queries now that a payment channel is created: + + * Check the watcher logs for received payments along with the GQL queries: + + ```bash + # Expected output: + # ... + # laconic:payments Received a payment voucher of 50 from 0x86804299822212c070178B5135Ba6DdAcFC357D3 + # laconic:payments Serving a paid query for 0x86804299822212c070178B5135Ba6DdAcFC357D3 + # vulcanize:resolver isPhisher 0x98ae4f9e9d01cc892adfe6871e1db0287039e0c183d3b5bb31d724228c114744 0x2B6AFbd4F479cE4101Df722cF4E05F941523EaD9 TWT:ash1 + # vulcanize:indexer isPhisher: db miss, fetching from upstream server + # laconic:payments Making RPC call: eth_chainId + # laconic:payments Making RPC call: eth_getBlockByHash + # 2023-10-13T06:33:58.443Z ts-nitro:engine{"msg":"Sent message","_msg":{"to":"0xAAA662","from":"0xBBB676", "payloadSummaries":[],"proposalSummaries":[],"payments":[{"amount":200,"channelId":"0x9a4bdfe03c8e72f368aab07e6bd18f1cd6821170e1a93e59864aad6ee6a853a2"}],"rejectedObjectives":[]}} + # laconic:payments Making RPC call: eth_chainId + # laconic:payments Making RPC call: eth_getStorageAt + # 2023-10-13T06:34:00.483Z ts-nitro:engine {"msg":"Sent message","_msg":{"to":"0xAAA662","from":"0xBBB676",# "payloadSummaries":[],"proposalSummaries":[],"payments":[{"amount":300,"channelId":"0x9a4bdfe03c8e72f368aab07e6bd18f1cd6821170e1a93e59864aad6ee6a853a2"}],"rejectedObjectives":[]}} + ``` + + * The watcher makes several ETH RPC requests to `ipld-eth-server-1` to fetch data required for satisfying the GQL request(s); check `ipld-eth-server-1` logs for charged RPC requests (`eth_getBlockByHash`, `eth_getBlockByNumber`, `eth_getStorageAt`): + + ```bash + # Expected output: + # ... + # 2023/10/13 06:34:57 INFO Received a voucher payer=0xBBB676f9cFF8D242e9eaC39D063848807d3D1D94 amount=100 + # 2023/10/13 06:34:59 INFO Serving a paid RPC request method=eth_getBlockByHash cost=50 sender=0xBBB676f9cFF8D242e9eaC39D063848807d3D1D94 + # time="2023-10-13T06:34:59Z" level=debug msg=START api_method=eth_getBlockByHash api_params="[0x46411a554f30e607bdd79cd157a60a7c8b314c0709557be3357ed2fef53d6c3d false]" api_reqid=52 conn="192.168.64.4:59644" user_id= uuid=a1893211-6992-11ee-a67a-0242c0a8400c + # WARN [10-13|06:34:59.133] Attempting GerRPCCalls, but default PluginLoader has not been initialized + # time="2023-10-13T06:34:59Z" level=debug msg=END api_method=eth_getBlockByHash api_params="[0x46411a554f30e607bdd79cd157a60a7c8b314c0709557be3357ed2fef53d6c3d false]" api_reqid=52 conn="192.168.64.4:59644" duration=5 user_id= uuid=a1893211-6992-11ee-a67a-0242c0a8400c + # 2023/10/13 06:34:59 INFO Serving a free RPC request method=eth_chainId + # time="2023-10-13T06:34:59Z" level=debug msg=START api_method=eth_chainId api_params="[]" api_reqid=53 conn="192.168.64.4:59658" user_id= uuid=a18a96d9-6992-11ee-a67a-0242c0a8400c + # time="2023-10-13T06:34:59Z" level=debug msg=END api_method=eth_chainId api_params="[]" api_reqid=53 conn="192.168.64.4:59658" duration=0 user_id= uuid=a18a96d9-6992-11ee-a67a-0242c0a8400c + # 2023/10/13 06:34:59 INFO Received a voucher payer=0xBBB676f9cFF8D242e9eaC39D063848807d3D1D94 amount=100 + # 2023/10/13 06:35:01 INFO Serving a paid RPC request method=eth_getStorageAt cost=50 sender=0xBBB676f9cFF8D242e9eaC39D063848807d3D1D94 + # time="2023-10-13T06:35:01Z" level=debug msg=START api_method=eth_getStorageAt api_params="[0x2b6afbd4f479ce4101df722cf4e05f941523ead9 0xf76621c2cc1c1705fabbc22d30ffc5ded7f26e4995feb8d1c9812a2697ba1278 0x46411a554f30e607bdd79cd157a60a7c8b314c0709557be3357ed2fef53d6c3d]" api_reqid=54 conn="192.168.64.4:59660" user_id= uuid=a2bd75d2-6992-11ee-a67a-0242c0a8400c + # time="2023-10-13T06:35:01Z" level=debug msg=END api_method=eth_getStorageAt api_params="[0x2b6afbd4f479ce4101df722cf4e05f941523ead9 0xf76621c2cc1c1705fabbc22d30ffc5ded7f26e4995feb8d1c9812a2697ba1278 0x46411a554f30e607bdd79cd157a60a7c8b314c0709557be3357ed2fef53d6c3d]" api_reqid=54 conn="192.168.64.4:59660" duration=7 user_id= uuid=a2bd75d2-6992-11ee-a67a-0242c0a8400c + ``` + +* Change the amount besides `PAY` button in debug panel to `>=100` for phisher reports next + +* Perform a phisher report and check the watcher logs for received payments: + + ```bash + # Expected output: + # ... + # vulcanize:libp2p-utils [6:50:2] Received a message on mobymask P2P network from peer: 12D3KooWRkxV9SX8uTUZYkbRjai4Fsn7yavB61J5TMnksixsabsP + # ts-nitro:engine {"msg":"Received message","_msg":{"to":"0xBBB676","from":"0x868042","payloadSummaries":[],"proposalSummaries":[],"payments":[{"amount":200,"channelId":"0x557153d729cf3323c0bdb40a36b245f98c2d4562933ba2182c9d61c5cfeda948"}],"rejectedObjectives":[]}} + # laconic:payments Received a payment voucher of 100 from 0x86804299822212c070178B5135Ba6DdAcFC357D3 + # vulcanize:libp2p-utils Payment received for a mutation request from 0x86804299822212c070178B5135Ba6DdAcFC357D3 + # vulcanize:libp2p-utils Transaction receipt for invoke message { + # to: '0x2B6AFbd4F479cE4101Df722cF4E05F941523EaD9', + # blockNumber: 232, + # blockHash: '0x6a188722c102662ea48af3786fe9db0d4b6c7ab7b27473eb0e628cf95746a244', + # transactionHash: '0x6521205db8a905b3222adc2b6855f9b2abc72580624d299bec2a35bcba173efa', + # effectiveGasPrice: '1500000007', + # gasUsed: '113355' + # } + ``` + +* Check the watcher - ipld-eth-server-1 payment channel status after a few requests: + + ```bash + docker exec payments-nitro-rpc-client-1 npm exec -c "nitro-rpc-client get-payment-channel $WATCHER_UPSTREAM_PAYMENT_CHANNEL -s false -h ipld-eth-server-1 -p 4005" + + # Expected output ('PaidSoFar' should be non zero): + # { + # ID: '0x8c0d17639bd2ba07dbcd248304a8f3c6c7276bfe25c2b87fe41f461e20f33f01', + # Status: 'Open', + # Balance: { + # AssetAddress: '0x0000000000000000000000000000000000000000', + # Payee: '0xaaa6628ec44a8a742987ef3a114ddfe2d4f7adce', + # Payer: '0xbbb676f9cff8d242e9eac39d063848807d3d1d94', + # PaidSoFar: 30000n, + # RemainingFunds: 999970000n + # } + # } + ``` + + +## Clean Up + +* In the MobyMask app, perform `VIRTUAL DEFUND` and `DIRECT DEFUND` (in order) for closing the payment channel created with watcher + +* Run the following in the browser console to delete the Nitro node's data: + + ```bash + await clearNodeStorage() + ``` + +* Run the following in the browser console to clear data in local storage: + ```bash + localStorage.clear() + ``` + +* On a fresh restart, clear activity tab data in MetaMask for concerned accounts diff --git a/stack_orchestrator/data/stacks/fixturenet-payments/ponder-demo.md b/stack_orchestrator/data/stacks/fixturenet-payments/ponder-demo.md new file mode 100644 index 00000000..2ea416b6 --- /dev/null +++ b/stack_orchestrator/data/stacks/fixturenet-payments/ponder-demo.md @@ -0,0 +1,267 @@ +# Demo + +Stack components: +* `ipld-eth-db` database for statediffed data +* Local geth + lighthouse blockchain "fixturenet" running in statediffing mode +* `ipld-eth-server-2` which runs an ETH RPC API and a GQL server; they both serve data from `ipld-eth-db` +* A go-nitro deployment acting as the remote Nitro node for `ipld-eth-server-2` +* Example ERC20 Ponder apps + * `ponder-app-indexer-1` that pays the `ipld-eth-server-2` for ETH RPC requests + * `ponder-app-indexer-2` that pays `ponder-app-indexer-1` for GQL queries + * `ponder-app-watcher` that pays `ponder-app-indexer-2` for GQL queries + +## Setup + +* In a terminal, check `ipld-eth-server-2`'s logs to keep track of incoming RPC requests from the `ponder-app-indexer-1`: + + ```bash + docker logs -f $(docker ps -aq --filter name="ipld-eth-server-2") + ``` + +## Run + +### ERC20 Ponder App + +* Run the first indexer Ponder app: + + ```bash + docker exec -it payments-ponder-app-indexer-1-1 bash -c "DEBUG=laconic:payments pnpm start" + + # Expected output: + # 12:57:03.751 INFO payment Nitro node setup with address 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d + # laconic:payments Starting voucher subscription... +0ms + # ... + # 09:58:54.288 INFO payment Creating ledger channel with nitro node 0x660a4bEF3fbC863Fcd8D3CDB39242aE513d7D92e ... + # 09:59:14.230 INFO payment Creating payment channel with nitro node 0x660a4bEF3fbC863Fcd8D3CDB39242aE513d7D92e ... + # 09:59:14.329 INFO payment Using payment channel 0x1ff59db391b7a55bed723b930ab53c80e7ce857487c1e58771aa5a0737d71625 + ``` + +* Export the payment channel id to a variable: + + ```bash + export PONDER_UPSTREAM_PAYMENT_CHANNEL= + ``` + +* On starting the Ponder app in indexer mode, it creates a payment channel with the `ipld-eth-server-2`'s (external) Nitro node and then starts the historical sync service + +* The sync service makes several ETH RPC requests to the `ipld-eth-server-2` to fetch required data; check the `ipld-eth-server-2` logs for charged RPC requests (`eth_getBlockByNumber`, `eth_getLogs`): + + ```bash + # Expected output: + # ... + # 2023/10/18 05:29:38 INFO Serving a paid RPC request method=eth_getBlockByNumber cost=50 sender=0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d + # time="2023-10-18T05:29:38Z" level=debug msg=START api_method=eth_getBlockByNumber api_params="[latest true]" api_reqid=0 conn="172.26.0.19:42306" user_id= uuid=54992dc3-6d77-11ee-9ede-0242ac1a000f + # WARN [10-18|05:29:38.292] Attempting GerRPCCalls, but default PluginLoader has not been initialized + # time="2023-10-18T05:29:38Z" level=debug msg=END api_method=eth_getBlockByNumber api_params="[latest true]" api_reqid=0 conn="172.26.0.19:42306" duration=22 user_id= uuid=54992dc3-6d77-11ee-9ede-0242ac1a000f + # ... + # 2023/10/18 05:29:40 INFO Serving a paid RPC request method=eth_getLogs cost=50 sender=0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d + # WARN [10-18|05:29:40.340] Attempting GerRPCCalls, but default PluginLoader has not been initialized + # time="2023-10-18T05:29:40Z" level=debug msg="retrieving log cids for receipt ids" + # ... + ``` + +* Check the Ponder - ipld-eth-server-2 payment channel status: + + ```bash + docker exec payments-nitro-rpc-client-1 npm exec -c "nitro-rpc-client get-payment-channel $PONDER_UPSTREAM_PAYMENT_CHANNEL -s false -h go-nitro -p 4006" + + # Expected output ('PaidSoFar' is non zero): + # { + # ID: '0x1ff59db391b7a55bed723b930ab53c80e7ce857487c1e58771aa5a0737d71625', + # Status: 'Open', + # Balance: { + # AssetAddress: '0x0000000000000000000000000000000000000000', + # Payee: '0x660a4bef3fbc863fcd8d3cdb39242ae513d7d92e', + # Payer: '0x67d5b55604d1af90074fcb69b8c51838fff84f8d', + # PaidSoFar: 7200n, + # RemainingFunds: 999992800n + # } + # } + ``` + +* In another terminal run the second indexer Ponder app: + + ```bash + docker exec -it payments-ponder-app-indexer-2-1 bash -c "DEBUG=laconic:payments pnpm start" + + # Expected output: + # 08:00:28.701 INFO payment Nitro node setup with address 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 + # laconic:payments Starting voucher subscription... +0ms + # ... + # 09:58:54.288 INFO payment Creating ledger channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ... + # 09:59:14.230 INFO payment Creating payment channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ... + # 09:59:14.329 INFO payment Using payment channel 0xfbf9d7eb7c18446883c7f57f4c94db5607f414a224b3e921c787db07371d2a70 + ``` + +* On starting indexer Ponder app, it creates a payment channel with first indexer and then starts the sync services + +* Check logs in `ponder-indexer-1` to see payments made from `ponder-indexer-2` from GQL queries to fetch network data + + ```bash + # ... + # laconic:payments Received a payment voucher of 100 from 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 +23ms + # laconic:payments Serving a paid query for 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 +0ms + # 13:01:05.671 DEBUG payment Verified payment for GQL queries getEthLogs + # laconic:payments Received a payment voucher of 100 from 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 +20ms + # laconic:payments Serving a paid query for 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 +0ms + # 13:01:05.691 DEBUG payment Verified payment for GQL queries getEthBlock + # 13:01:07.598 INFO realtime Fetched missing blocks [686, 687] (network=fixturenet) + # 13:01:07.598 DEBUG realtime Started processing new head block 686 (network=fixturenet) + # ... + ``` + +* Following are the example GQL queries (with payment headers) `ponder-indexer-2` makes to `ponder-indexer-1` for fetching data + + ```graphql + { + getEthLogs( + chainId: 1212, + address: "0x6325439389e0797ab35752b4f43a14c004f22a9c", + fromBlock: 100, + toBlock: 200, + ) { + address + blockHash + blockNumber + data + logIndex + removed + topics + transactionHash + transactionIndex + } + + getEthBlock( + chainId: 1212, + blockNumber: 150, + fullTransactions: false + ) { + baseFeePerGas + difficulty + extraData + gasLimit + gasUsed + hash + logsBloom + miner + mixHash + nonce + number + parentHash + receiptsRoot + sha3Uncles + size + stateRoot + timestamp + totalDifficulty + transactionsRoot + txHashes + } + } + ``` + +* In another terminal run the Ponder app in watcher mode: + + ```bash + docker exec -it payments-ponder-app-watcher-1 bash -c "DEBUG=laconic:payments pnpm start" + + # Expected output: + # 11:23:22.057 DEBUG app Started using config file: ponder.config.ts + # 08:02:12.548 INFO payment Nitro node setup with address 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db + # laconic:payments Starting voucher subscription... +0ms + # 08:02:17.417 INFO payment Creating ledger channel with nitro node 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 ... + # 08:02:37.135 INFO payment Creating payment channel with nitro node 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 ... + # 08:02:37.313 INFO payment Using payment channel 0xc48622577dfa389283beb19ed946274eb034587d72e61445dc997304be671f1a + # ... + # 11:23:22.436 INFO server Started responding as healthy + ``` + +* Check the terminal of the second indexer Ponder app. Logs of payment for `getLogEvents` queries can be seen: + + ```bash + # ... + # 08:02:37.763 DEBUG realtime Finished processing new head block 89 (network=fixturenet) + # laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +444ms + # laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +1ms + # 08:02:37.804 DEBUG payment Verified payment for GQL queries getLogEvents + # laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +45ms + # laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +0ms + # 08:02:37.849 DEBUG payment Verified payment for GQL queries getLogEvents + ``` + +* Following are the example GQL queries (with payment headers) `ponder-watcher` makes to `ponder-indexer-2` for fetching data + + ```graphql + getLogEvents( + fromTimestamp: 1695303079, + toTimestamp: 1695304156, + filters: [ + { + name: "AdventureGold" + chainId: 1212, + address: ["0x6325439389e0797ab35752b4f43a14c004f22a9c"], + fromBlock: 100, + toBlock: 200, + includeEventSelectors: ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"] + } + ] + ) { + events { + transaction { + hash + } + log { + id + } + block { + number + } + } + metadata { + pageEndsAtTimestamp + isLastPage + } + } + ``` + +* Open watcher Ponder app endpoint http://localhost:42069 + + * Try GQL query to see transfer events + + ```graphql + { + transferEvents (orderBy: "timestamp", orderDirection: "desc") { + id + amount + from { + id + } + to { + id + } + timestamp + } + } + ``` + +* Transfer an ERC20 token on chain + + * Get the deployed ERC20 token address + + ```bash + export TOKEN_ADDRESS=$(docker exec payments-ponder-er20-contracts-1 jq -r '.address' ./deployment/erc20-address.json) + ``` + + * Transfer token + + ```bash + docker exec -it payments-ponder-er20-contracts-1 bash -c "yarn token:transfer:docker --token ${TOKEN_ADDRESS} --to 0xe22AD83A0dE117bA0d03d5E94Eb4E0d80a69C62a --amount 5000" + ``` + +* Wait for a log in watcher Ponder app + + ```bash + 06:40:47.567 INFO handlers Processed 1 event (up to Oct 19, 2023) + ``` + +* Check the GQL query again in http://localhost:42069 to see a new `TransferEvent` entity diff --git a/app/data/stacks/fixturenet-payments/stack.yml b/stack_orchestrator/data/stacks/fixturenet-payments/stack.yml similarity index 66% rename from app/data/stacks/fixturenet-payments/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-payments/stack.yml index ec52b577..e13d303c 100644 --- a/app/data/stacks/fixturenet-payments/stack.yml +++ b/stack_orchestrator/data/stacks/fixturenet-payments/stack.yml @@ -6,19 +6,22 @@ repos: - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5 - git.vdb.to/cerc-io/lighthouse - git.vdb.to/cerc-io/ipld-eth-db@v5 - - git.vdb.to/cerc-io/ipld-eth-server@v1.11.6-statediff-v5 - # nitro repos - - github.com/cerc-io/ts-nitro@v0.1.13 - - github.com/cerc-io/go-nitro@v0.1.1-ts-port-0.1.5 + # ipld-eth-server with payments -- to be used by ipld-eth-server-payments pod + - git.vdb.to/cerc-io/ipld-eth-server@payments + # nitro repo + - github.com/cerc-io/ts-nitro@v0.1.15 + - github.com/cerc-io/go-nitro@v0.1.2-ts-port-0.1.9 # mobymask watcher repos - - github.com/cerc-io/watcher-ts@v0.2.63 - - github.com/cerc-io/mobymask-v2-watcher-ts@v0.2.2 + - github.com/cerc-io/watcher-ts@v0.2.66 + # this is mobymask-v3 + - github.com/cerc-io/mobymask-v2-watcher-ts@v0.2.4 - github.com/cerc-io/MobyMask@v0.1.3 # mobymask app repos - github.com/cerc-io/mobymask-snap - - github.com/cerc-io/mobymask-ui@v0.2.1 + - github.com/cerc-io/mobymask-ui@v0.2.2 # ponder repo - github.com/cerc-io/ponder@laconic-esm + - github.com/cerc-io/erc20-watcher-ts containers: # fixturenet images - cerc/go-ethereum @@ -41,14 +44,16 @@ containers: - cerc/mobymask-ui # ponder image - cerc/ponder + - cerc/watcher-erc20 pods: - fixturenet-eth - - ipld-eth-server + - ipld-eth-server-payments - ipld-eth-db - nitro-contracts - go-nitro - - nitro-reverse-payment-proxy + - nitro-rpc-client - watcher-mobymask-v3 - mobymask-snap - mobymask-app-v3 - - ponder + - ponder-indexer + - ponder-watcher diff --git a/app/data/stacks/fixturenet-plugeth-tx/README.md b/stack_orchestrator/data/stacks/fixturenet-plugeth-tx/README.md similarity index 100% rename from app/data/stacks/fixturenet-plugeth-tx/README.md rename to stack_orchestrator/data/stacks/fixturenet-plugeth-tx/README.md diff --git a/app/data/stacks/fixturenet-plugeth-tx/stack.yml b/stack_orchestrator/data/stacks/fixturenet-plugeth-tx/stack.yml similarity index 100% rename from app/data/stacks/fixturenet-plugeth-tx/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-plugeth-tx/stack.yml diff --git a/app/data/stacks/fixturenet-pocket/README.md b/stack_orchestrator/data/stacks/fixturenet-pocket/README.md similarity index 100% rename from app/data/stacks/fixturenet-pocket/README.md rename to stack_orchestrator/data/stacks/fixturenet-pocket/README.md diff --git a/app/data/stacks/fixturenet-pocket/stack.yml b/stack_orchestrator/data/stacks/fixturenet-pocket/stack.yml similarity index 100% rename from app/data/stacks/fixturenet-pocket/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-pocket/stack.yml diff --git a/app/data/stacks/fixturenet-sushiswap-subgraph/README.md b/stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/README.md similarity index 100% rename from app/data/stacks/fixturenet-sushiswap-subgraph/README.md rename to stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/README.md diff --git a/app/data/stacks/fixturenet-sushiswap-subgraph/stack.yml b/stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/stack.yml similarity index 100% rename from app/data/stacks/fixturenet-sushiswap-subgraph/stack.yml rename to stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/stack.yml diff --git a/app/data/stacks/gelato/README.md b/stack_orchestrator/data/stacks/gelato/README.md similarity index 100% rename from app/data/stacks/gelato/README.md rename to stack_orchestrator/data/stacks/gelato/README.md diff --git a/app/data/stacks/gelato/stack.yml b/stack_orchestrator/data/stacks/gelato/stack.yml similarity index 100% rename from app/data/stacks/gelato/stack.yml rename to stack_orchestrator/data/stacks/gelato/stack.yml diff --git a/app/data/stacks/graph-node/README.md b/stack_orchestrator/data/stacks/graph-node/README.md similarity index 100% rename from app/data/stacks/graph-node/README.md rename to stack_orchestrator/data/stacks/graph-node/README.md diff --git a/app/data/stacks/graph-node/deploy-subgraph.md b/stack_orchestrator/data/stacks/graph-node/deploy-subgraph.md similarity index 100% rename from app/data/stacks/graph-node/deploy-subgraph.md rename to stack_orchestrator/data/stacks/graph-node/deploy-subgraph.md diff --git a/app/data/stacks/graph-node/stack.yml b/stack_orchestrator/data/stacks/graph-node/stack.yml similarity index 100% rename from app/data/stacks/graph-node/stack.yml rename to stack_orchestrator/data/stacks/graph-node/stack.yml diff --git a/app/data/stacks/kubo/README.md b/stack_orchestrator/data/stacks/kubo/README.md similarity index 100% rename from app/data/stacks/kubo/README.md rename to stack_orchestrator/data/stacks/kubo/README.md diff --git a/app/data/stacks/kubo/stack.yml b/stack_orchestrator/data/stacks/kubo/stack.yml similarity index 100% rename from app/data/stacks/kubo/stack.yml rename to stack_orchestrator/data/stacks/kubo/stack.yml diff --git a/stack_orchestrator/data/stacks/laconic-dot-com/README.md b/stack_orchestrator/data/stacks/laconic-dot-com/README.md new file mode 100644 index 00000000..3e7ad328 --- /dev/null +++ b/stack_orchestrator/data/stacks/laconic-dot-com/README.md @@ -0,0 +1,9 @@ +# laconic-dot-com + +``` +laconic-so --stack laconic-dot-com setup-repositories +laconic-so --stack laconic-dot-com build-containers +laconic-so --stack laconic-dot-com deploy init --output laconic-website-spec.yml --map-ports-to-host localhost-same +laconic-so --stack laconic-dot-com deploy create --spec-file laconic-website-spec.yml --deployment-dir lx-website +laconic-so deployment --dir lx-website start +``` diff --git a/stack_orchestrator/data/stacks/laconic-dot-com/stack.yml b/stack_orchestrator/data/stacks/laconic-dot-com/stack.yml new file mode 100644 index 00000000..2bc98952 --- /dev/null +++ b/stack_orchestrator/data/stacks/laconic-dot-com/stack.yml @@ -0,0 +1,8 @@ +version: "0.1" +name: laconic-dot-com +repos: + - github.com/LaconicNetwork/laconic.com@v0.0.1 +containers: + - cerc/laconic-dot-com +pods: + - laconic-dot-com diff --git a/app/data/stacks/lasso/README.md b/stack_orchestrator/data/stacks/lasso/README.md similarity index 100% rename from app/data/stacks/lasso/README.md rename to stack_orchestrator/data/stacks/lasso/README.md diff --git a/app/data/stacks/lasso/stack.yml b/stack_orchestrator/data/stacks/lasso/stack.yml similarity index 100% rename from app/data/stacks/lasso/stack.yml rename to stack_orchestrator/data/stacks/lasso/stack.yml diff --git a/stack_orchestrator/data/stacks/mainnet-eth-plugeth/README.md b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/README.md new file mode 100644 index 00000000..8ed6bebb --- /dev/null +++ b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/README.md @@ -0,0 +1,141 @@ +# mainnet-eth-plugeth + +Deploys a "head-tracking" mainnet Ethereum stack comprising a [plugeth](https://git.vdb.to/cerc-io/plugeth) execution layer node and a [lighthouse](https://github.com/sigp/lighthouse) consensus layer node, with [plugeth-statediff](https://git.vdb.to/cerc-io/plugeth-statediff) for statediffing, [ipld-eth-db](https://git.vdb.to/cerc-io/ipld-eth-db) for storage, and [ipld-eth-server](https://git.vdb.to/cerc-io/ipld-eth-server) for indexed ETH IPLD objects. + +## Clone required repositories + +``` +$ laconic-so --stack mainnet-eth-plugeth setup-repositories +``` + +## Build containers + +``` +$ laconic-so --stack mainnet-eth-plugeth build-containers +``` + +## Create a deployment + +``` +$ laconic-so --stack mainnet-eth-plugeth deploy init --map-ports-to-host any-same --output mainnet-eth-plugeth-spec.yml +$ laconic-so --stack mainnet-eth-plugeth deploy create --spec-file mainnet-eth-plugeth-spec.yml --deployment-dir mainnet-eth-plugeth-deployment +``` +## Start the stack +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment start +``` +Display stack status: +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment ps +Running containers: +id: f39608eca04d72d6b0f1f3acefc5ebb52908da06e221d20c7138f7e3dff5e423, name: laconic-ef641b4d13eb61ed561b19be67063241-foundry-1, ports: +id: 4052b1eddd886ae0d6b41f9ff22e68a70f267b2bfde10f4b7b79b5bd1eeddcac, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-geth-1-1, ports: 30303/tcp, 30303/udp, 0.0.0.0:49184->40000/tcp, 0.0.0.0:49185->6060/tcp, 0.0.0.0:49186->8545/tcp, 8546/tcp +id: ac331232e597944b621b3b8942ace5dafb14524302cab338ff946c7f6e5a1d52, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1, ports: 0.0.0.0:49187->8001/tcp +``` +See stack logs: +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment logs +time="2023-07-25T09:46:29-06:00" level=warning msg="The \"CERC_SCRIPT_DEBUG\" variable is not set. Defaulting to a blank string." +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.362 INFO Logging to file path: "/var/lighthouse-data-dir/beacon/logs/beacon.log" +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.365 INFO Lighthouse started version: Lighthouse/v4.1.0-693886b +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.365 INFO Configured for network name: mainnet +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.366 INFO Data directory initialised datadir: /var/lighthouse-data-dir +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.366 INFO Deposit contract address: 0x00000000219ab540356cbb839cbe05303d7705fa, deploy_block: 11184524 +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.424 INFO Starting checkpoint sync remote_url: https://beaconstate.ethstaker.cc/, service: beacon +``` +## Monitoring stack sync progress +Both go-ethereum and lighthouse will engage in an initial chain sync phase that will last up to several hours depending on hardware performance and network capacity. +Syncing can be monitored by looking for these log messages: +``` +Jul 24 12:34:17.001 INFO Downloading historical blocks est_time: 5 days 11 hrs, speed: 14.67 slots/sec, distance: 6932481 slots (137 weeks 3 days), service: slot_notifier +INFO [07-24|12:14:52.493] Syncing beacon headers downloaded=145,920 left=17,617,968 eta=1h23m32.815s +INFO [07-24|12:33:15.238] Syncing: chain download in progress synced=1.86% chain=148.94MiB headers=368,640@95.03MiB bodies=330,081@40.56MiB receipts=330,081@13.35MiB eta=37m54.505s +INFO [07-24|12:35:13.028] Syncing: state download in progress synced=1.32% state=4.64GiB accounts=2,850,314@677.57MiB slots=18,663,070@3.87GiB codes=26662@111.14MiB eta=3h18m0.699s +``` +Once synced up these log messages will be observed: +``` +INFO Synced slot: 6952515, block: 0x5bcb…f6d9, epoch: 217266, finalized_epoch: 217264, finalized_root: 0x6342…2c5c, exec_hash: 0x8d8c…2443 (verified), peers: 31, service: slot_notifier +INFO [07-25|03:04:48.941] Imported new potential chain segment number=17,767,316 hash=84f6e7..bc2cb0 blocks=1 txs=137 mgas=16.123 elapsed=57.087ms mgasps=282.434 dirty=461.46MiB +INFO [07-25|03:04:49.042] Chain head was updated number=17,767,316 hash=84f6e7..bc2cb0 root=ca58b2..8258c1 elapsed=2.480111ms +``` +## Clean up + +Stop the stack: +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment stop +``` +This leaves data volumes in place, allowing the stack to be subsequently re-started. +To permanently *delete* the stack's data volumes run: +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment stop --delete-data-volumes +``` +After deleting the volumes, any subsequent re-start will begin chain sync from cold. + +## Ports +It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections. +Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be +customized by editing the "spec" file generated by `laconic-so deploy init`. + +In this example, ports `8545` and `5052` have been assigned to a specific addresses/port combination on the host, while +port `40000` has been left with random assignment: +``` +$ cat mainnet-eth-plugeth-spec.yml +stack: mainnet-eth-plugeth +ports: + mainnet-eth-plugeth-geth-1: + - '10.10.10.10:8545:8545' + - '40000' + mainnet-eth-plugeth-lighthouse-1: + - '10.10.10.10:5052:5052' +volumes: + mainnet_eth_plugeth_config_data: ./data/mainnet_eth_plugeth_config_data + mainnet_eth_plugeth_geth_1_data: ./data/mainnet_eth_plugeth_geth_1_data + mainnet_eth_plugeth_lighthouse_1_data: ./data/mainnet_eth_plugeth_lighthouse_1_data +``` +In addition, a stack-wide port mapping "recipe" can be applied at the time the +`laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: +| Recipe | Host Port Mapping | +|--------|-------------------| +| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) | +| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers | +| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers | +| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)| +| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) | +## Data volumes +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory: +``` +$ cat mainnet-eth-plugeth-spec.yml +stack: mainnet-eth-plugeth +ports: + mainnet-eth-plugeth-geth-1: + - '10.10.10.10:8545:8545' + - '40000' + mainnet-eth-plugeth-lighthouse-1: + - '10.10.10.10:5052:5052' +volumes: + mainnet_eth_plugeth_config_data: ./data/mainnet_eth_plugeth_config_data + mainnet_eth_plugeth_geth_1_data: ./data/mainnet_eth_plugeth_geth_1_data + mainnet_eth_plugeth_lighthouse_1_data: ./data/mainnet_eth_plugeth_lighthouse_1_data +``` +A synced-up stack will consume around 900GB of data volume space: +``` +$ sudo du -h mainnet-eth-plugeth-deployment/data/ +150M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/freezer_db +25G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/chain_db +16K mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/network +368M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/logs +26G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon +26G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data +8.0K mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_config_data +4.0K mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/keystore +527G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata/ancient/chain +527G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata/ancient +859G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata +4.8M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/nodes +242M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/ethash +669M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/triecache +860G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth +860G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data +885G mainnet-eth-plugeth-deployment/data/ +``` diff --git a/stack_orchestrator/data/stacks/mainnet-eth-plugeth/deploy/commands.py b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/deploy/commands.py new file mode 100644 index 00000000..5aba9547 --- /dev/null +++ b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/deploy/commands.py @@ -0,0 +1,32 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from secrets import token_hex + + +def init(ctx): + return None + + +def setup(ctx): + return None + + +def create(ctx, extra_args): + # Generate the JWT secret and save to its config file + secret = token_hex(32) + jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_plugeth_config_data", "jwtsecret") + with open(jwt_file_path, 'w+') as jwt_file: + jwt_file.write(secret) diff --git a/stack_orchestrator/data/stacks/mainnet-eth-plugeth/stack.yml b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/stack.yml new file mode 100644 index 00000000..7ade244c --- /dev/null +++ b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/stack.yml @@ -0,0 +1,29 @@ +version: "1.2" +name: mainnet-eth +description: "Ethereum Mainnet" +repos: + - git.vdb.to/cerc-io/plugeth@statediff + - git.vdb.to/cerc-io/plugeth-statediff + - git.vdb.to/cerc-io/lighthouse + - git.vdb.to/cerc-io/ipld-eth-db@v5 + - git.vdb.to/cerc-io/ipld-eth-server@v5 + - git.vdb.to/cerc-io/keycloak-reg-api + - git.vdb.to/cerc-io/keycloak-reg-ui +containers: + - cerc/plugeth-statediff + - cerc/plugeth + - cerc/plugeth-with-plugins + - cerc/lighthouse + - cerc/lighthouse-cli + - cerc/ipld-eth-db + - cerc/ipld-eth-server + - cerc/keycloak + - cerc/webapp-base + - cerc/keycloak-reg-api + - cerc/keycloak-reg-ui +pods: + - mainnet-eth-plugeth + - mainnet-eth-ipld-eth-db + - mainnet-eth-ipld-eth-server + - mainnet-eth-keycloak + - mainnet-eth-metrics diff --git a/app/data/stacks/mainnet-eth/README.md b/stack_orchestrator/data/stacks/mainnet-eth/README.md similarity index 100% rename from app/data/stacks/mainnet-eth/README.md rename to stack_orchestrator/data/stacks/mainnet-eth/README.md diff --git a/app/data/stacks/mainnet-eth/deploy/commands.py b/stack_orchestrator/data/stacks/mainnet-eth/deploy/commands.py similarity index 100% rename from app/data/stacks/mainnet-eth/deploy/commands.py rename to stack_orchestrator/data/stacks/mainnet-eth/deploy/commands.py diff --git a/app/data/stacks/mainnet-eth/stack.yml b/stack_orchestrator/data/stacks/mainnet-eth/stack.yml similarity index 100% rename from app/data/stacks/mainnet-eth/stack.yml rename to stack_orchestrator/data/stacks/mainnet-eth/stack.yml diff --git a/app/data/stacks/mainnet-go-opera/README.md b/stack_orchestrator/data/stacks/mainnet-go-opera/README.md similarity index 100% rename from app/data/stacks/mainnet-go-opera/README.md rename to stack_orchestrator/data/stacks/mainnet-go-opera/README.md diff --git a/app/data/stacks/mainnet-go-opera/stack.yml b/stack_orchestrator/data/stacks/mainnet-go-opera/stack.yml similarity index 100% rename from app/data/stacks/mainnet-go-opera/stack.yml rename to stack_orchestrator/data/stacks/mainnet-go-opera/stack.yml diff --git a/app/data/stacks/mainnet-laconic/README.md b/stack_orchestrator/data/stacks/mainnet-laconic/README.md similarity index 100% rename from app/data/stacks/mainnet-laconic/README.md rename to stack_orchestrator/data/stacks/mainnet-laconic/README.md diff --git a/app/data/stacks/mainnet-laconic/deploy/commands.py b/stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py similarity index 97% rename from app/data/stacks/mainnet-laconic/deploy/commands.py rename to stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py index 3a62b42d..b611a0d6 100644 --- a/app/data/stacks/mainnet-laconic/deploy/commands.py +++ b/stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py @@ -13,11 +13,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from app.util import get_yaml -from app.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext -from app.stack_state import State -from app.deploy_util import VolumeMapping, run_container_command -from app.command_types import CommandOptions +from stack_orchestrator.util import get_yaml +from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext +from stack_orchestrator.deploy.stack_state import State +from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command +from stack_orchestrator.command_types import CommandOptions from enum import Enum from pathlib import Path from shutil import copyfile, copytree diff --git a/app/data/stacks/mainnet-laconic/stack.yml b/stack_orchestrator/data/stacks/mainnet-laconic/stack.yml similarity index 100% rename from app/data/stacks/mainnet-laconic/stack.yml rename to stack_orchestrator/data/stacks/mainnet-laconic/stack.yml diff --git a/app/data/stacks/mainnet-laconic/test/run-mainnet-laconic-test.sh b/stack_orchestrator/data/stacks/mainnet-laconic/test/run-mainnet-laconic-test.sh similarity index 100% rename from app/data/stacks/mainnet-laconic/test/run-mainnet-laconic-test.sh rename to stack_orchestrator/data/stacks/mainnet-laconic/test/run-mainnet-laconic-test.sh diff --git a/app/data/stacks/mobymask-v2/README.md b/stack_orchestrator/data/stacks/mobymask-v2/README.md similarity index 100% rename from app/data/stacks/mobymask-v2/README.md rename to stack_orchestrator/data/stacks/mobymask-v2/README.md diff --git a/app/data/stacks/mobymask-v2/demo.md b/stack_orchestrator/data/stacks/mobymask-v2/demo.md similarity index 100% rename from app/data/stacks/mobymask-v2/demo.md rename to stack_orchestrator/data/stacks/mobymask-v2/demo.md diff --git a/app/data/stacks/mobymask-v2/mobymask-only.md b/stack_orchestrator/data/stacks/mobymask-v2/mobymask-only.md similarity index 100% rename from app/data/stacks/mobymask-v2/mobymask-only.md rename to stack_orchestrator/data/stacks/mobymask-v2/mobymask-only.md diff --git a/app/data/stacks/mobymask-v2/stack.yml b/stack_orchestrator/data/stacks/mobymask-v2/stack.yml similarity index 100% rename from app/data/stacks/mobymask-v2/stack.yml rename to stack_orchestrator/data/stacks/mobymask-v2/stack.yml diff --git a/app/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md b/stack_orchestrator/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md similarity index 100% rename from app/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md rename to stack_orchestrator/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md diff --git a/app/data/stacks/mobymask-v2/watcher-p2p-network/web-app.md b/stack_orchestrator/data/stacks/mobymask-v2/watcher-p2p-network/web-app.md similarity index 100% rename from app/data/stacks/mobymask-v2/watcher-p2p-network/web-app.md rename to stack_orchestrator/data/stacks/mobymask-v2/watcher-p2p-network/web-app.md diff --git a/app/data/stacks/mobymask-v2/web-apps.md b/stack_orchestrator/data/stacks/mobymask-v2/web-apps.md similarity index 100% rename from app/data/stacks/mobymask-v2/web-apps.md rename to stack_orchestrator/data/stacks/mobymask-v2/web-apps.md diff --git a/app/data/stacks/mobymask-v3/README.md b/stack_orchestrator/data/stacks/mobymask-v3/README.md similarity index 100% rename from app/data/stacks/mobymask-v3/README.md rename to stack_orchestrator/data/stacks/mobymask-v3/README.md diff --git a/app/data/stacks/mobymask-v3/stack.yml b/stack_orchestrator/data/stacks/mobymask-v3/stack.yml similarity index 100% rename from app/data/stacks/mobymask-v3/stack.yml rename to stack_orchestrator/data/stacks/mobymask-v3/stack.yml diff --git a/app/data/stacks/mobymask-v3/watcher.md b/stack_orchestrator/data/stacks/mobymask-v3/watcher.md similarity index 100% rename from app/data/stacks/mobymask-v3/watcher.md rename to stack_orchestrator/data/stacks/mobymask-v3/watcher.md diff --git a/app/data/stacks/mobymask-v3/web-app.md b/stack_orchestrator/data/stacks/mobymask-v3/web-app.md similarity index 100% rename from app/data/stacks/mobymask-v3/web-app.md rename to stack_orchestrator/data/stacks/mobymask-v3/web-app.md diff --git a/app/data/stacks/mobymask/README.md b/stack_orchestrator/data/stacks/mobymask/README.md similarity index 100% rename from app/data/stacks/mobymask/README.md rename to stack_orchestrator/data/stacks/mobymask/README.md diff --git a/app/data/stacks/mobymask/stack.yml b/stack_orchestrator/data/stacks/mobymask/stack.yml similarity index 100% rename from app/data/stacks/mobymask/stack.yml rename to stack_orchestrator/data/stacks/mobymask/stack.yml diff --git a/app/data/stacks/package-registry/README.md b/stack_orchestrator/data/stacks/package-registry/README.md similarity index 100% rename from app/data/stacks/package-registry/README.md rename to stack_orchestrator/data/stacks/package-registry/README.md diff --git a/app/data/stacks/package-registry/stack.yml b/stack_orchestrator/data/stacks/package-registry/stack.yml similarity index 70% rename from app/data/stacks/package-registry/stack.yml rename to stack_orchestrator/data/stacks/package-registry/stack.yml index 33c6c939..f6367ab1 100644 --- a/app/data/stacks/package-registry/stack.yml +++ b/stack_orchestrator/data/stacks/package-registry/stack.yml @@ -13,3 +13,8 @@ pods: path: gitea pre_start_command: "run-this-first.sh" post_start_command: "initialize-gitea.sh" + - name: act-runner + repository: cerc-io/hosting + path: act-runner + pre_start_command: "pre_start.sh" + post_start_command: "post_start.sh" diff --git a/app/data/stacks/reth/README.md b/stack_orchestrator/data/stacks/reth/README.md similarity index 100% rename from app/data/stacks/reth/README.md rename to stack_orchestrator/data/stacks/reth/README.md diff --git a/app/data/stacks/reth/stack.yml b/stack_orchestrator/data/stacks/reth/stack.yml similarity index 100% rename from app/data/stacks/reth/stack.yml rename to stack_orchestrator/data/stacks/reth/stack.yml diff --git a/app/data/stacks/sushiswap-subgraph/README.md b/stack_orchestrator/data/stacks/sushiswap-subgraph/README.md similarity index 100% rename from app/data/stacks/sushiswap-subgraph/README.md rename to stack_orchestrator/data/stacks/sushiswap-subgraph/README.md diff --git a/app/data/stacks/sushiswap-subgraph/stack.yml b/stack_orchestrator/data/stacks/sushiswap-subgraph/stack.yml similarity index 100% rename from app/data/stacks/sushiswap-subgraph/stack.yml rename to stack_orchestrator/data/stacks/sushiswap-subgraph/stack.yml diff --git a/app/data/stacks/sushiswap/README.md b/stack_orchestrator/data/stacks/sushiswap/README.md similarity index 100% rename from app/data/stacks/sushiswap/README.md rename to stack_orchestrator/data/stacks/sushiswap/README.md diff --git a/app/data/stacks/sushiswap/smoke-tests.md b/stack_orchestrator/data/stacks/sushiswap/smoke-tests.md similarity index 100% rename from app/data/stacks/sushiswap/smoke-tests.md rename to stack_orchestrator/data/stacks/sushiswap/smoke-tests.md diff --git a/app/data/stacks/sushiswap/stack.yml b/stack_orchestrator/data/stacks/sushiswap/stack.yml similarity index 100% rename from app/data/stacks/sushiswap/stack.yml rename to stack_orchestrator/data/stacks/sushiswap/stack.yml diff --git a/app/data/stacks/test/README.md b/stack_orchestrator/data/stacks/test/README.md similarity index 100% rename from app/data/stacks/test/README.md rename to stack_orchestrator/data/stacks/test/README.md diff --git a/app/data/stacks/test/deploy/commands.py b/stack_orchestrator/data/stacks/test/deploy/commands.py similarity index 88% rename from app/data/stacks/test/deploy/commands.py rename to stack_orchestrator/data/stacks/test/deploy/commands.py index fbb9a8ea..e6601eae 100644 --- a/app/data/stacks/test/deploy/commands.py +++ b/stack_orchestrator/data/stacks/test/deploy/commands.py @@ -13,10 +13,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from app.util import get_yaml -from app.deploy_types import DeployCommandContext -from app.stack_state import State -from app.deploy_util import VolumeMapping, run_container_command +from stack_orchestrator.util import get_yaml +from stack_orchestrator.deploy.deploy_types import DeployCommandContext +from stack_orchestrator.deploy.stack_state import State +from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command from pathlib import Path default_spec_file_content = """config: diff --git a/app/data/stacks/test/stack.yml b/stack_orchestrator/data/stacks/test/stack.yml similarity index 83% rename from app/data/stacks/test/stack.yml rename to stack_orchestrator/data/stacks/test/stack.yml index a62da193..ac724c89 100644 --- a/app/data/stacks/test/stack.yml +++ b/stack_orchestrator/data/stacks/test/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: test description: "A test stack" repos: - - github.com/cerc-io/laconicd + - git.vdb.to/cerc-io/laconicd - git.vdb.to/cerc-io/test-project@test-branch containers: - cerc/test-container diff --git a/app/data/stacks/uniswap-v3/README.md b/stack_orchestrator/data/stacks/uniswap-v3/README.md similarity index 100% rename from app/data/stacks/uniswap-v3/README.md rename to stack_orchestrator/data/stacks/uniswap-v3/README.md diff --git a/app/data/stacks/uniswap-v3/stack.yml b/stack_orchestrator/data/stacks/uniswap-v3/stack.yml similarity index 100% rename from app/data/stacks/uniswap-v3/stack.yml rename to stack_orchestrator/data/stacks/uniswap-v3/stack.yml diff --git a/app/data/version.txt b/stack_orchestrator/data/version.txt similarity index 100% rename from app/data/version.txt rename to stack_orchestrator/data/version.txt diff --git a/stack_orchestrator/deploy/__init__.py b/stack_orchestrator/deploy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stack_orchestrator/deploy/compose/__init__.py b/stack_orchestrator/deploy/compose/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py new file mode 100644 index 00000000..79ab1482 --- /dev/null +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -0,0 +1,79 @@ +# Copyright © 2022, 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from pathlib import Path +from python_on_whales import DockerClient, DockerException +from stack_orchestrator.deploy.deployer import Deployer, DeployerException, DeployerConfigGenerator + + +class DockerDeployer(Deployer): + name: str = "compose" + + def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name, + compose_env_file=compose_env_file) + + def up(self, detach, services): + try: + return self.docker.compose.up(detach=detach, services=services) + except DockerException as e: + raise DeployerException(e) + + def down(self, timeout, volumes): + try: + return self.docker.compose.down(timeout=timeout, volumes=volumes) + except DockerException as e: + raise DeployerException(e) + + def ps(self): + try: + return self.docker.compose.ps() + except DockerException as e: + raise DeployerException(e) + + def port(self, service, private_port): + try: + return self.docker.compose.port(service=service, private_port=private_port) + except DockerException as e: + raise DeployerException(e) + + def execute(self, service, command, tty, envs): + try: + return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs) + except DockerException as e: + raise DeployerException(e) + + def logs(self, services, tail, follow, stream): + try: + return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream) + except DockerException as e: + raise DeployerException(e) + + def run(self, image, command, user, volumes, entrypoint=None): + try: + return self.docker.run(image=image, command=command, user=user, volumes=volumes, entrypoint=entrypoint) + except DockerException as e: + raise DeployerException(e) + + +class DockerDeployerConfigGenerator(DeployerConfigGenerator): + config_file_name: str = "kind-config.yml" + + def __init__(self) -> None: + super().__init__() + + # Nothing needed at present for the docker deployer + def generate(self, deployment_dir: Path): + pass diff --git a/app/deploy.py b/stack_orchestrator/deploy/deploy.py similarity index 82% rename from app/deploy.py rename to stack_orchestrator/deploy/deploy.py index 0ebcfe18..1c467067 100644 --- a/app/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -13,7 +13,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -# Deploys the system components using docker-compose +# Deploys the system components using a deployer (either docker-compose or k8s) import hashlib import copy @@ -22,14 +22,16 @@ import sys from dataclasses import dataclass from importlib import resources import subprocess -from python_on_whales import DockerClient, DockerException import click from pathlib import Path -from app.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path -from app.deploy_types import ClusterContext, DeployCommandContext -from app.deployment_create import create as deployment_create -from app.deployment_create import init as deployment_init -from app.deployment_create import setup as deployment_setup +from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path +from stack_orchestrator.deploy.deployer import Deployer, DeployerException +from stack_orchestrator.deploy.deployer_factory import getDeployer +from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext +from stack_orchestrator.deploy.deployment_context import DeploymentContext +from stack_orchestrator.deploy.deployment_create import create as deployment_create +from stack_orchestrator.deploy.deployment_create import init as deployment_init +from stack_orchestrator.deploy.deployment_create import setup as deployment_setup @click.group() @@ -37,8 +39,9 @@ from app.deployment_create import setup as deployment_setup @click.option("--exclude", help="don\'t start these components") @click.option("--env-file", help="env file to be used") @click.option("--cluster", help="specify a non-default cluster name") +@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s)") @click.pass_context -def command(ctx, include, exclude, env_file, cluster): +def command(ctx, include, exclude, env_file, cluster, deploy_to): '''deploy a stack''' # Although in theory for some subcommands (e.g. deploy create) the stack can be inferred, @@ -50,16 +53,23 @@ def command(ctx, include, exclude, env_file, cluster): if ctx.parent.obj.debug: print(f"ctx.parent.obj: {ctx.parent.obj}") - ctx.obj = create_deploy_context(global_options2(ctx), stack, include, exclude, cluster, env_file) + + if deploy_to is None: + deploy_to = "compose" + + ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to) # Subcommand is executed now, by the magic of click -def create_deploy_context(global_context, stack, include, exclude, cluster, env_file): +def create_deploy_context( + global_context, deployment_context: DeploymentContext, stack, include, exclude, cluster, env_file, deployer): cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file) + deployment_dir = deployment_context.deployment_dir if deployment_context else None # See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/ - docker = DockerClient(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster, - compose_env_file=cluster_context.env_file) - return DeployCommandContext(stack, cluster_context, docker) + deployer = getDeployer(deployer, deployment_dir, compose_files=cluster_context.compose_files, + compose_project_name=cluster_context.cluster, + compose_env_file=cluster_context.env_file) + return DeployCommandContext(stack, cluster_context, deployer) def up_operation(ctx, services_list, stay_attached=False): @@ -74,10 +84,10 @@ def up_operation(ctx, services_list, stay_attached=False): print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}") for pre_start_command in cluster_context.pre_start_commands: _run_command(global_context, cluster_context.cluster, pre_start_command) - deploy_context.docker.compose.up(detach=not stay_attached, services=services_list) + deploy_context.deployer.up(detach=not stay_attached, services=services_list) for post_start_command in cluster_context.post_start_commands: _run_command(global_context, cluster_context.cluster, post_start_command) - _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.docker, container_exec_env) + _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env) def down_operation(ctx, delete_volumes, extra_args_list): @@ -89,7 +99,7 @@ def down_operation(ctx, delete_volumes, extra_args_list): if extra_args_list: timeout_arg = extra_args_list[0] # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully - ctx.obj.docker.compose.down(timeout=timeout_arg, volumes=delete_volumes) + ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes) def ps_operation(ctx): @@ -97,7 +107,7 @@ def ps_operation(ctx): if not global_context.dry_run: if global_context.verbose: print("Running compose ps") - container_list = ctx.obj.docker.compose.ps() + container_list = ctx.obj.deployer.ps() if len(container_list) > 0: print("Running containers:") for container in container_list: @@ -128,7 +138,7 @@ def port_operation(ctx, extra_args): exposed_port = extra_args_list[1] if global_context.verbose: print(f"Running compose port {service_name} {exposed_port}") - mapped_port_data = ctx.obj.docker.compose.port(service_name, exposed_port) + mapped_port_data = ctx.obj.deployer.port(service_name, exposed_port) print(f"{mapped_port_data[0]}:{mapped_port_data[1]}") @@ -145,8 +155,8 @@ def exec_operation(ctx, extra_args): if global_context.verbose: print(f"Running compose exec {service_name} {command_to_exec}") try: - ctx.obj.docker.compose.execute(service_name, command_to_exec, envs=container_exec_env) - except DockerException: + ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env) + except DeployerException: print("container command returned error exit status") @@ -157,7 +167,7 @@ def logs_operation(ctx, tail: int, follow: bool, extra_args: str): if global_context.verbose: print("Running compose logs") services_list = extra_args_list if extra_args_list is not None else [] - logs_stream = ctx.obj.docker.compose.logs(services=services_list, tail=tail, follow=follow, stream=True) + logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True) for stream_type, stream_content in logs_stream: print(stream_content.decode("utf-8"), end="") @@ -214,11 +224,11 @@ def get_stack_status(ctx, stack): ctx_copy.stack = stack cluster_context = _make_cluster_context(ctx_copy, stack, None, None, None, None) - docker = DockerClient(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster) + deployer = Deployer(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster) # TODO: refactor to avoid duplicating this code above if ctx.verbose: print("Running compose ps") - container_list = docker.compose.ps() + container_list = deployer.ps() if len(container_list) > 0: if ctx.debug: print(f"Container list from compose ps: {container_list}") @@ -252,7 +262,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): deployment = True else: # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose") + compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose") if cluster is None: # Create default unique, stable cluster name from confile file path and stack name if provided @@ -267,7 +277,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): print(f"Using cluster name: {cluster}") # See: https://stackoverflow.com/a/20885799/1701505 - from app import data + from stack_orchestrator import data with resources.open_text(data, "pod-list.txt") as pod_list_file: all_pods = pod_list_file.read().splitlines() @@ -301,7 +311,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml") else: if deployment: - compose_file_name = os.path.join(compose_dir, "docker-compose.yml") + compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml") pod_pre_start_command = pod["pre_start_command"] pod_post_start_command = pod["post_start_command"] script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts") @@ -311,7 +321,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): post_start_commands.append(os.path.join(script_dir, pod_post_start_command)) else: pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"]) - compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml") + compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml") pod_pre_start_command = pod["pre_start_command"] pod_post_start_command = pod["post_start_command"] if pod_pre_start_command is not None: @@ -359,7 +369,7 @@ def _run_command(ctx, cluster_name, command): sys.exit(1) -def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env): +def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_env): @dataclass class ConfigDirective: @@ -390,13 +400,13 @@ def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env) # TODO: fix the script paths so they're consistent between containers source_value = None try: - source_value = docker.compose.execute(pd.source_container, - ["sh", "-c", - "sh /docker-entrypoint-scripts.d/export-" - f"{pd.source_variable}.sh"], - tty=False, - envs=container_exec_env) - except DockerException as error: + source_value = deployer.execute(pd.source_container, + ["sh", "-c", + "sh /docker-entrypoint-scripts.d/export-" + f"{pd.source_variable}.sh"], + tty=False, + envs=container_exec_env) + except DeployerException as error: if ctx.debug: print(f"Docker exception reading config source: {error}") # If the script executed failed for some reason, we get: @@ -411,12 +421,12 @@ def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env) if source_value: if ctx.debug: print(f"fetched source value: {source_value}") - destination_output = docker.compose.execute(pd.destination_container, - ["sh", "-c", - f"sh /scripts/import-{pd.destination_variable}.sh" - f" {source_value}"], - tty=False, - envs=container_exec_env) + destination_output = deployer.execute(pd.destination_container, + ["sh", "-c", + f"sh /scripts/import-{pd.destination_variable}.sh" + f" {source_value}"], + tty=False, + envs=container_exec_env) waiting_for_data = False if ctx.debug: print(f"destination output: {destination_output}") diff --git a/app/deploy_types.py b/stack_orchestrator/deploy/deploy_types.py similarity index 85% rename from app/deploy_types.py rename to stack_orchestrator/deploy/deploy_types.py index 63f32762..f97b2649 100644 --- a/app/deploy_types.py +++ b/stack_orchestrator/deploy/deploy_types.py @@ -13,11 +13,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import List +from typing import List, Mapping from dataclasses import dataclass -from pathlib import Path -from python_on_whales import DockerClient -from app.command_types import CommandOptions +from stack_orchestrator.command_types import CommandOptions +from stack_orchestrator.deploy.deployer import Deployer @dataclass @@ -35,13 +34,7 @@ class ClusterContext: class DeployCommandContext: stack: str cluster_context: ClusterContext - docker: DockerClient - - -@dataclass -class DeploymentContext: - deployment_dir: Path - command_context: DeployCommandContext + deployer: Deployer @dataclass @@ -66,3 +59,8 @@ class LaconicStackSetupCommand: @dataclass class LaconicStackCreateCommand: network_dir: str + + +@dataclass +class DeployEnvVars: + map: Mapping[str, str] diff --git a/app/deploy_util.py b/stack_orchestrator/deploy/deploy_util.py similarity index 90% rename from app/deploy_util.py rename to stack_orchestrator/deploy/deploy_util.py index 498e3dfd..9829490d 100644 --- a/app/deploy_util.py +++ b/stack_orchestrator/deploy/deploy_util.py @@ -15,8 +15,8 @@ import os from typing import List -from app.deploy_types import DeployCommandContext, VolumeMapping -from app.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list +from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping +from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list def _container_image_from_service(stack: str, service: str): @@ -47,12 +47,12 @@ def _volumes_to_docker(mounts: List[VolumeMapping]): def run_container_command(ctx: DeployCommandContext, service: str, command: str, mounts: List[VolumeMapping]): - docker = ctx.docker + deployer = ctx.deployer container_image = _container_image_from_service(ctx.stack, service) docker_volumes = _volumes_to_docker(mounts) if ctx.cluster_context.options.debug: print(f"Running this command in {service} container: {command}") - docker_output = docker.run( + docker_output = deployer.run( container_image, ["-c", command], entrypoint="sh", user=f"{os.getuid()}:{os.getgid()}", diff --git a/stack_orchestrator/deploy/deployer.py b/stack_orchestrator/deploy/deployer.py new file mode 100644 index 00000000..68b0088a --- /dev/null +++ b/stack_orchestrator/deploy/deployer.py @@ -0,0 +1,60 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from abc import ABC, abstractmethod +from pathlib import Path + + +class Deployer(ABC): + + @abstractmethod + def up(self, detach, services): + pass + + @abstractmethod + def down(self, timeout, volumes): + pass + + @abstractmethod + def ps(self): + pass + + @abstractmethod + def port(self, service, private_port): + pass + + @abstractmethod + def execute(self, service_name, command, tty, envs): + pass + + @abstractmethod + def logs(self, services, tail, follow, stream): + pass + + @abstractmethod + def run(self, image, command, user, volumes, entrypoint): + pass + + +class DeployerException(Exception): + def __init__(self, *args: object) -> None: + super().__init__(*args) + + +class DeployerConfigGenerator(ABC): + + @abstractmethod + def generate(self, deployment_dir: Path): + pass diff --git a/stack_orchestrator/deploy/deployer_factory.py b/stack_orchestrator/deploy/deployer_factory.py new file mode 100644 index 00000000..5d515418 --- /dev/null +++ b/stack_orchestrator/deploy/deployer_factory.py @@ -0,0 +1,35 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from stack_orchestrator.deploy.k8s.deploy_k8s import K8sDeployer, K8sDeployerConfigGenerator +from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer, DockerDeployerConfigGenerator + + +def getDeployerConfigGenerator(type: str): + if type == "compose" or type is None: + return DockerDeployerConfigGenerator() + elif type == "k8s": + return K8sDeployerConfigGenerator() + else: + print(f"ERROR: deploy-to {type} is not valid") + + +def getDeployer(type: str, deployment_dir, compose_files, compose_project_name, compose_env_file): + if type == "compose" or type is None: + return DockerDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file) + elif type == "k8s": + return K8sDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file) + else: + print(f"ERROR: deploy-to {type} is not valid") diff --git a/app/deployment.py b/stack_orchestrator/deploy/deployment.py similarity index 84% rename from app/deployment.py rename to stack_orchestrator/deploy/deployment.py index aeabf61c..e22d7dcc 100644 --- a/app/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -14,33 +14,18 @@ # along with this program. If not, see . import click -from dataclasses import dataclass from pathlib import Path import sys -from app.deploy import up_operation, down_operation, ps_operation, port_operation -from app.deploy import exec_operation, logs_operation, create_deploy_context - - -@dataclass -class DeploymentContext: - dir: Path - - def get_stack_file(self): - return self.dir.joinpath("stack.yml") - - def get_env_file(self): - return self.dir.joinpath("config.env") - - # TODO: implement me - def get_cluster_name(self): - return None +from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation +from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context +from stack_orchestrator.deploy.deployment_context import DeploymentContext @click.group() @click.option("--dir", required=True, help="path to deployment directory") @click.pass_context def command(ctx, dir): - '''create a deployment''' + '''manage a deployment''' # Check that --stack wasn't supplied if ctx.parent.obj.stack: @@ -55,14 +40,18 @@ def command(ctx, dir): print(f"Error: supplied deployment directory path {dir} exists but is a file not a directory") sys.exit(1) # Store the deployment context for subcommands - ctx.obj = DeploymentContext(dir_path) + deployment_context = DeploymentContext() + deployment_context.init(dir_path) + ctx.obj = deployment_context def make_deploy_context(ctx): - stack_file_path = ctx.obj.get_stack_file() - env_file = ctx.obj.get_env_file() - cluster_name = ctx.obj.get_cluster_name() - return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, cluster_name, env_file) + context: DeploymentContext = ctx.obj + stack_file_path = context.get_stack_file() + env_file = context.get_env_file() + cluster_name = context.get_cluster_name() + return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file, + context.spec.obj["deploy-to"]) @command.command() diff --git a/stack_orchestrator/deploy/deployment_context.py b/stack_orchestrator/deploy/deployment_context.py new file mode 100644 index 00000000..cd731394 --- /dev/null +++ b/stack_orchestrator/deploy/deployment_context.py @@ -0,0 +1,46 @@ + +# Copyright © 2022, 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from pathlib import Path + +from stack_orchestrator.deploy.stack import Stack +from stack_orchestrator.deploy.spec import Spec + + +class DeploymentContext: + deployment_dir: Path + spec: Spec + stack: Stack + + def get_stack_file(self): + return self.deployment_dir.joinpath("stack.yml") + + def get_spec_file(self): + return self.deployment_dir.joinpath("spec.yml") + + def get_env_file(self): + return self.deployment_dir.joinpath("config.env") + + # TODO: implement me + def get_cluster_name(self): + return None + + def init(self, dir): + self.deployment_dir = dir + self.spec = Spec() + self.spec.init_from_file(self.get_spec_file()) + self.stack = Stack(self.spec.obj["stack"]) + self.stack.init_from_file(self.get_stack_file()) diff --git a/app/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py similarity index 82% rename from app/deployment_create.py rename to stack_orchestrator/deploy/deployment_create.py index 7f416300..c00c0dc6 100644 --- a/app/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -21,9 +21,12 @@ from typing import List import random from shutil import copy, copyfile, copytree import sys -from app.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml, - get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_path) -from app.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand +from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, + global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts, + get_pod_script_paths, get_plugin_code_paths) +from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand +from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator +from stack_orchestrator.deploy.deployment_context import DeploymentContext def _make_default_deployment_dir(): @@ -106,9 +109,10 @@ def _fixup_pod_file(pod, spec, compose_dir): pod["services"][container_name]["ports"] = container_ports -def _commands_plugin_path(ctx: DeployCommandContext): - plugin_path = get_plugin_code_path(ctx.stack) - return plugin_path.joinpath("deploy", "commands.py") +def _commands_plugin_paths(stack_name: str): + plugin_paths = get_plugin_code_paths(stack_name) + ret = [p.joinpath("deploy", "commands.py") for p in plugin_paths] + return ret # See: https://stackoverflow.com/a/54625079/1701505 @@ -120,15 +124,23 @@ def call_stack_deploy_init(deploy_command_context): # Link with the python file in the stack # Call a function in it # If no function found, return None - python_file_path = _commands_plugin_path(deploy_command_context) - if python_file_path.exists(): - spec = util.spec_from_file_location("commands", python_file_path) - imported_stack = util.module_from_spec(spec) - spec.loader.exec_module(imported_stack) - if _has_method(imported_stack, "init"): - return imported_stack.init(deploy_command_context) - else: - return None + python_file_paths = _commands_plugin_paths(deploy_command_context.stack) + + ret = None + init_done = False + for python_file_path in python_file_paths: + if python_file_path.exists(): + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + if _has_method(imported_stack, "init"): + if not init_done: + ret = imported_stack.init(deploy_command_context) + init_done = True + else: + # TODO: remove this restriction + print(f"Skipping init() from plugin {python_file_path}. Only one init() is allowed.") + return ret # TODO: fold this with function above @@ -136,16 +148,14 @@ def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetu # Link with the python file in the stack # Call a function in it # If no function found, return None - python_file_path = _commands_plugin_path(deploy_command_context) - print(f"Path: {python_file_path}") - if python_file_path.exists(): - spec = util.spec_from_file_location("commands", python_file_path) - imported_stack = util.module_from_spec(spec) - spec.loader.exec_module(imported_stack) - if _has_method(imported_stack, "setup"): - return imported_stack.setup(deploy_command_context, parameters, extra_args) - else: - return None + python_file_paths = _commands_plugin_paths(deploy_command_context.stack) + for python_file_path in python_file_paths: + if python_file_path.exists(): + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + if _has_method(imported_stack, "setup"): + imported_stack.setup(deploy_command_context, parameters, extra_args) # TODO: fold this with function above @@ -153,15 +163,14 @@ def call_stack_deploy_create(deployment_context, extra_args): # Link with the python file in the stack # Call a function in it # If no function found, return None - python_file_path = _commands_plugin_path(deployment_context.command_context) - if python_file_path.exists(): - spec = util.spec_from_file_location("commands", python_file_path) - imported_stack = util.module_from_spec(spec) - spec.loader.exec_module(imported_stack) - if _has_method(imported_stack, "create"): - return imported_stack.create(deployment_context, extra_args) - else: - return None + python_file_paths = _commands_plugin_paths(deployment_context.stack.name) + for python_file_path in python_file_paths: + if python_file_path.exists(): + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + if _has_method(imported_stack, "create"): + imported_stack.create(deployment_context, extra_args) # Inspect the pod yaml to find config files referenced in subdirectories @@ -249,13 +258,13 @@ def init(ctx, config, output, map_ports_to_host): stack = global_options(ctx).stack debug = global_options(ctx).debug default_spec_file_content = call_stack_deploy_init(ctx.obj) - spec_file_content = {"stack": stack} + spec_file_content = {"stack": stack, "deploy-to": ctx.obj.deployer.name} if default_spec_file_content: spec_file_content.update(default_spec_file_content) config_variables = _parse_config_variables(config) if config_variables: # Implement merge, since update() overwrites - orig_config = spec_file_content["config"] + orig_config = spec_file_content.get("config", {}) new_config = config_variables["config"] merged_config = {**new_config, **orig_config} spec_file_content.update({"config": merged_config}) @@ -303,7 +312,7 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path): def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): # This function fails with a useful error message if the file doens't exist parsed_spec = get_parsed_deployment_spec(spec_file) - stack_name = parsed_spec['stack'] + stack_name = parsed_spec["stack"] stack_file = get_stack_file_path(stack_name) parsed_stack = get_parsed_stack_config(stack_name) if global_options(ctx).debug: @@ -315,7 +324,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): sys.exit(1) os.mkdir(deployment_dir) # Copy spec file and the stack file into the deployment dir - copyfile(spec_file, os.path.join(deployment_dir, os.path.basename(spec_file))) + copyfile(spec_file, os.path.join(deployment_dir, "spec.yml")) copyfile(stack_file, os.path.join(deployment_dir, os.path.basename(stack_file))) # Copy any config varibles from the spec file into an env file suitable for compose _write_config_file(spec_file, os.path.join(deployment_dir, "config.env")) @@ -325,7 +334,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): os.mkdir(destination_compose_dir) destination_pods_dir = os.path.join(deployment_dir, "pods") os.mkdir(destination_pods_dir) - data_dir = Path(__file__).absolute().parent.joinpath("data") + data_dir = Path(__file__).absolute().parent.parent.joinpath("data") yaml = get_yaml() for pod in pods: pod_file_path = get_pod_file_path(parsed_stack, pod) @@ -336,7 +345,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): if global_options(ctx).debug: print(f"extra config dirs: {extra_config_dirs}") _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir) - with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file: + with open(os.path.join(destination_compose_dir, "docker-compose-%s.yml" % pod), "w") as output_file: yaml.dump(parsed_pod_file, output_file) # Copy the config files for the pod, if any config_dirs = {pod} @@ -359,7 +368,12 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): # stack member here. deployment_command_context = ctx.obj deployment_command_context.stack = stack_name - deployment_context = DeploymentContext(Path(deployment_dir), deployment_command_context) + deployment_context = DeploymentContext() + deployment_context.init(Path(deployment_dir)) + # Call the deployer to generate any deployer-specific files (e.g. for kind) + deployer_config_generator = getDeployerConfigGenerator(parsed_spec["deploy-to"]) + # TODO: make deployment_dir a Path above + deployer_config_generator.generate(Path(deployment_dir)) call_stack_deploy_create(deployment_context, [network_dir, initial_peers]) diff --git a/stack_orchestrator/deploy/k8s/__init__.py b/stack_orchestrator/deploy/k8s/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py new file mode 100644 index 00000000..9275db2b --- /dev/null +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -0,0 +1,131 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from kubernetes import client +from typing import Any, List, Set + +from stack_orchestrator.opts import opts +from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files +from stack_orchestrator.deploy.k8s.helpers import parsed_pod_files_map_from_file_names, get_node_pv_mount_path +from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map +from stack_orchestrator.deploy.deploy_types import DeployEnvVars + + +class ClusterInfo: + parsed_pod_yaml_map: Any = {} + image_set: Set[str] = set() + app_name: str = "test-app" + deployment_name: str = "test-deployment" + environment_variables: DeployEnvVars + + def __init__(self) -> None: + pass + + def int(self, pod_files: List[str], compose_env_file): + self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) + # Find the set of images in the pods + for pod_name in self.parsed_pod_yaml_map: + pod = self.parsed_pod_yaml_map[pod_name] + services = pod["services"] + for service_name in services: + service_info = services[service_name] + image = service_info["image"] + self.image_set.add(image) + if opts.o.debug: + print(f"image_set: {self.image_set}") + self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) + if (opts.o.debug): + print(f"Env vars: {self.environment_variables.map}") + + def get_pvcs(self): + result = [] + volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + if opts.o.debug: + print(f"Volumes: {volumes}") + for volume_name in volumes: + spec = client.V1PersistentVolumeClaimSpec( + access_modes=["ReadWriteOnce"], + storage_class_name="manual", + resources=client.V1ResourceRequirements( + requests={"storage": "2Gi"} + ), + volume_name=volume_name + ) + pvc = client.V1PersistentVolumeClaim( + metadata=client.V1ObjectMeta(name=volume_name, + labels={"volume-label": volume_name}), + spec=spec, + ) + result.append(pvc) + return result + + def get_pvs(self): + result = [] + volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + for volume_name in volumes: + spec = client.V1PersistentVolumeSpec( + storage_class_name="manual", + access_modes=["ReadWriteOnce"], + capacity={"storage": "2Gi"}, + host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name)) + ) + pv = client.V1PersistentVolume( + metadata=client.V1ObjectMeta(name=volume_name, + labels={"volume-label": volume_name}), + spec=spec, + ) + result.append(pv) + return result + + # to suit the deployment, and also annotate the container specs to point at said volumes + def get_deployment(self): + containers = [] + for pod_name in self.parsed_pod_yaml_map: + pod = self.parsed_pod_yaml_map[pod_name] + services = pod["services"] + for service_name in services: + container_name = service_name + service_info = services[service_name] + image = service_info["image"] + volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name) + container = client.V1Container( + name=container_name, + image=image, + env=envs_from_environment_variables_map(self.environment_variables.map), + ports=[client.V1ContainerPort(container_port=80)], + volume_mounts=volume_mounts, + resources=client.V1ResourceRequirements( + requests={"cpu": "100m", "memory": "200Mi"}, + limits={"cpu": "500m", "memory": "500Mi"}, + ), + ) + containers.append(container) + volumes = volumes_for_pod_files(self.parsed_pod_yaml_map) + template = client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta(labels={"app": self.app_name}), + spec=client.V1PodSpec(containers=containers, volumes=volumes), + ) + spec = client.V1DeploymentSpec( + replicas=1, template=template, selector={ + "matchLabels": + {"app": self.app_name}}) + + deployment = client.V1Deployment( + api_version="apps/v1", + kind="Deployment", + metadata=client.V1ObjectMeta(name=self.deployment_name), + spec=spec, + ) + return deployment diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py new file mode 100644 index 00000000..bc256b6b --- /dev/null +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -0,0 +1,143 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from pathlib import Path +from kubernetes import client, config + +from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator +from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind +from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config +from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo +from stack_orchestrator.opts import opts + + +class K8sDeployer(Deployer): + name: str = "k8s" + core_api: client.CoreV1Api + apps_api: client.AppsV1Api + k8s_namespace: str = "default" + kind_cluster_name: str + cluster_info : ClusterInfo + deployment_dir: Path + + def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + if (opts.o.debug): + print(f"Deployment dir: {deployment_dir}") + print(f"Compose files: {compose_files}") + print(f"Project name: {compose_project_name}") + print(f"Env file: {compose_env_file}") + self.deployment_dir = deployment_dir + self.kind_cluster_name = compose_project_name + self.cluster_info = ClusterInfo() + self.cluster_info.int(compose_files, compose_env_file) + + def connect_api(self): + config.load_kube_config(context=f"kind-{self.kind_cluster_name}") + self.core_api = client.CoreV1Api() + self.apps_api = client.AppsV1Api() + + def up(self, detach, services): + # Create the kind cluster + create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath("kind-config.yml")) + self.connect_api() + # Ensure the referenced containers are copied into kind + load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) + + # Create the host-path-mounted PVs for this deployment + pvs = self.cluster_info.get_pvs() + for pv in pvs: + if opts.o.debug: + print(f"Sending this pv: {pv}") + pv_resp = self.core_api.create_persistent_volume(body=pv) + if opts.o.debug: + print("PVs created:") + print(f"{pv_resp}") + + # Figure out the PVCs for this deployment + pvcs = self.cluster_info.get_pvcs() + for pvc in pvcs: + if opts.o.debug: + print(f"Sending this pvc: {pvc}") + pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace) + if opts.o.debug: + print("PVCs created:") + print(f"{pvc_resp}") + # Process compose files into a Deployment + deployment = self.cluster_info.get_deployment() + # Create the k8s objects + if opts.o.debug: + print(f"Sending this deployment: {deployment}") + deployment_resp = self.apps_api.create_namespaced_deployment( + body=deployment, namespace=self.k8s_namespace + ) + if opts.o.debug: + print("Deployment created:") + print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ + {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") + + def down(self, timeout, volumes): + # Delete the k8s objects + # Destroy the kind cluster + destroy_cluster(self.kind_cluster_name) + + def ps(self): + self.connect_api() + # Call whatever API we need to get the running container list + ret = self.core_api.list_pod_for_all_namespaces(watch=False) + if ret.items: + for i in ret.items: + print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name)) + ret = self.core_api.list_node(pretty=True, watch=False) + return [] + + def port(self, service, private_port): + # Since we handle the port mapping, need to figure out where this comes from + # Also look into whether it makes sense to get ports for k8s + pass + + def execute(self, service_name, command, tty, envs): + # Call the API to execute a command in a running container + pass + + def logs(self, services, tail, follow, stream): + self.connect_api() + pods = pods_in_deployment(self.core_api, "test-deployment") + if len(pods) > 1: + print("Warning: more than one pod in the deployment") + k8s_pod_name = pods[0] + log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test") + return log_stream_from_string(log_data) + + def run(self, image, command, user, volumes, entrypoint=None): + # We need to figure out how to do this -- check why we're being called first + pass + + +class K8sDeployerConfigGenerator(DeployerConfigGenerator): + config_file_name: str = "kind-config.yml" + + def __init__(self) -> None: + super().__init__() + + def generate(self, deployment_dir: Path): + # Check the file isn't already there + # Get the config file contents + content = generate_kind_config(deployment_dir) + if opts.o.debug: + print(f"kind config is: {content}") + config_file = deployment_dir.joinpath(self.config_file_name) + # Write the file + with open(config_file, "w") as output_file: + output_file.write(content) diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py new file mode 100644 index 00000000..db1ef075 --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -0,0 +1,241 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from kubernetes import client +from dotenv import dotenv_values +import os +from pathlib import Path +import subprocess +from typing import Any, Set, Mapping, List + +from stack_orchestrator.opts import opts +from stack_orchestrator.util import get_yaml + + +def _run_command(command: str): + if opts.o.debug: + print(f"Running: {command}") + result = subprocess.run(command, shell=True) + if opts.o.debug: + print(f"Result: {result}") + + +def create_cluster(name: str, config_file: str): + _run_command(f"kind create cluster --name {name} --config {config_file}") + + +def destroy_cluster(name: str): + _run_command(f"kind delete cluster --name {name}") + + +def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]): + for image in image_set: + _run_command(f"kind load docker-image {image} --name {kind_cluster_name}") + + +def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str): + pods = [] + pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app") + if opts.o.debug: + print(f"pod_response: {pod_response}") + for pod_info in pod_response.items: + pod_name = pod_info.metadata.name + pods.append(pod_name) + return pods + + +def log_stream_from_string(s: str): + # Note response has to be UTF-8 encoded because the caller expects to decode it + yield ("ignore", s.encode()) + + +def named_volumes_from_pod_files(parsed_pod_files): + # Parse the compose files looking for named volumes + named_volumes = [] + for pod in parsed_pod_files: + parsed_pod_file = parsed_pod_files[pod] + if "volumes" in parsed_pod_file: + volumes = parsed_pod_file["volumes"] + for volume in volumes.keys(): + # Volume definition looks like: + # 'laconicd-data': None + named_volumes.append(volume) + return named_volumes + + +def get_node_pv_mount_path(volume_name: str): + return f"/mnt/{volume_name}" + + +def volume_mounts_for_service(parsed_pod_files, service): + result = [] + # Find the service + for pod in parsed_pod_files: + parsed_pod_file = parsed_pod_files[pod] + if "services" in parsed_pod_file: + services = parsed_pod_file["services"] + for service_name in services: + if service_name == service: + service_obj = services[service_name] + if "volumes" in service_obj: + volumes = service_obj["volumes"] + for mount_string in volumes: + # Looks like: test-data:/data + (volume_name, mount_path) = mount_string.split(":") + volume_device = client.V1VolumeMount(mount_path=mount_path, name=volume_name) + result.append(volume_device) + return result + + +def volumes_for_pod_files(parsed_pod_files): + result = [] + for pod in parsed_pod_files: + parsed_pod_file = parsed_pod_files[pod] + if "volumes" in parsed_pod_file: + volumes = parsed_pod_file["volumes"] + for volume_name in volumes.keys(): + claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name) + volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim) + result.append(volume) + return result + + +def _get_host_paths_for_volumes(parsed_pod_files): + result = {} + for pod in parsed_pod_files: + parsed_pod_file = parsed_pod_files[pod] + if "volumes" in parsed_pod_file: + volumes = parsed_pod_file["volumes"] + for volume_name in volumes.keys(): + volume_definition = volumes[volume_name] + host_path = volume_definition["driver_opts"]["device"] + result[volume_name] = host_path + return result + + +def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Path: + if os.path.isabs(data_mount_path): + return data_mount_path + else: + # Python Path voodo that looks pretty odd: + return Path.cwd().joinpath(deployment_dir.joinpath("compose").joinpath(data_mount_path)).resolve() + + +def parsed_pod_files_map_from_file_names(pod_files): + parsed_pod_yaml_map : Any = {} + for pod_file in pod_files: + with open(pod_file, "r") as pod_file_descriptor: + parsed_pod_file = get_yaml().load(pod_file_descriptor) + parsed_pod_yaml_map[pod_file] = parsed_pod_file + if opts.o.debug: + print(f"parsed_pod_yaml_map: {parsed_pod_yaml_map}") + return parsed_pod_yaml_map + + +def _generate_kind_mounts(parsed_pod_files, deployment_dir): + volume_definitions = [] + volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files) + # Note these paths are relative to the location of the pod files (at present) + # So we need to fix up to make them correct and absolute because kind assumes + # relative to the cwd. + for pod in parsed_pod_files: + parsed_pod_file = parsed_pod_files[pod] + if "services" in parsed_pod_file: + services = parsed_pod_file["services"] + for service_name in services: + service_obj = services[service_name] + if "volumes" in service_obj: + volumes = service_obj["volumes"] + for mount_string in volumes: + # Looks like: test-data:/data + (volume_name, mount_path) = mount_string.split(":") + volume_definitions.append( + f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n" + f" containerPath: {get_node_pv_mount_path(volume_name)}" + ) + return ( + "" if len(volume_definitions) == 0 else ( + " extraMounts:\n" + f"{''.join(volume_definitions)}" + ) + ) + + +def _generate_kind_port_mappings(parsed_pod_files): + port_definitions = [] + for pod in parsed_pod_files: + parsed_pod_file = parsed_pod_files[pod] + if "services" in parsed_pod_file: + services = parsed_pod_file["services"] + for service_name in services: + service_obj = services[service_name] + if "ports" in service_obj: + ports = service_obj["ports"] + for port_string in ports: + # TODO handle the complex cases + # Looks like: 80 or something more complicated + port_definitions.append(f" - containerPort: {port_string}\n hostPort: {port_string}") + return ( + "" if len(port_definitions) == 0 else ( + " extraPortMappings:\n" + f"{''.join(port_definitions)}" + ) + ) + + +def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]: + result = [] + for env_var, env_val in map.items(): + result.append(client.V1EnvVar(env_var, env_val)) + return result + + +# This needs to know: +# The service ports for the cluster +# The bind mounted volumes for the cluster +# +# Make ports like this: +# extraPortMappings: +# - containerPort: 80 +# hostPort: 80 +# # optional: set the bind address on the host +# # 0.0.0.0 is the current default +# listenAddress: "127.0.0.1" +# # optional: set the protocol to one of TCP, UDP, SCTP. +# # TCP is the default +# protocol: TCP +# Make bind mounts like this: +# extraMounts: +# - hostPath: /path/to/my/files +# containerPath: /files +def generate_kind_config(deployment_dir: Path): + compose_file_dir = deployment_dir.joinpath("compose") + # TODO: this should come from the stack file, not this way + pod_files = [p for p in compose_file_dir.iterdir() if p.is_file()] + parsed_pod_files_map = parsed_pod_files_map_from_file_names(pod_files) + port_mappings_yml = _generate_kind_port_mappings(parsed_pod_files_map) + mounts_yml = _generate_kind_mounts(parsed_pod_files_map, deployment_dir) + return ( + "kind: Cluster\n" + "apiVersion: kind.x-k8s.io/v1alpha4\n" + "nodes:\n" + "- role: control-plane\n" + f"{port_mappings_yml}\n" + f"{mounts_yml}\n" + ) + + +def env_var_map_from_file(file: Path) -> Mapping[str, str]: + return dotenv_values(file) diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py new file mode 100644 index 00000000..9ee893b9 --- /dev/null +++ b/stack_orchestrator/deploy/spec.py @@ -0,0 +1,30 @@ +# Copyright © 2022, 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from pathlib import Path +import typing +from stack_orchestrator.util import get_yaml + + +class Spec: + + obj: typing.Any + + def __init__(self) -> None: + pass + + def init_from_file(self, file_path: Path): + with file_path: + self.obj = get_yaml().load(open(file_path, "r")) diff --git a/stack_orchestrator/deploy/stack.py b/stack_orchestrator/deploy/stack.py new file mode 100644 index 00000000..1a493534 --- /dev/null +++ b/stack_orchestrator/deploy/stack.py @@ -0,0 +1,31 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from pathlib import Path +import typing +from stack_orchestrator.util import get_yaml + + +class Stack: + + name: str + obj: typing.Any + + def __init__(self, name: str) -> None: + self.name = name + + def init_from_file(self, file_path: Path): + with file_path: + self.obj = get_yaml().load(open(file_path, "r")) diff --git a/app/stack_state.py b/stack_orchestrator/deploy/stack_state.py similarity index 100% rename from app/stack_state.py rename to stack_orchestrator/deploy/stack_state.py diff --git a/cli.py b/stack_orchestrator/main.py similarity index 72% rename from cli.py rename to stack_orchestrator/main.py index 63823715..0b0585e0 100644 --- a/cli.py +++ b/stack_orchestrator/main.py @@ -15,14 +15,16 @@ import click -from app.command_types import CommandOptions -from app import setup_repositories -from app import build_containers -from app import build_npms -from app import deploy -from app import version -from app import deployment -from app import update +from stack_orchestrator.command_types import CommandOptions +from stack_orchestrator.repos import setup_repositories +from stack_orchestrator.build import build_containers +from stack_orchestrator.build import build_npms +from stack_orchestrator.build import build_webapp +from stack_orchestrator.deploy import deploy +from stack_orchestrator import version +from stack_orchestrator.deploy import deployment +from stack_orchestrator import opts +from stack_orchestrator import update CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @@ -39,12 +41,15 @@ CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @click.pass_context def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error): """Laconic Stack Orchestrator""" - ctx.obj = CommandOptions(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error) + command_options = CommandOptions(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error) + opts.opts.o = command_options + ctx.obj = command_options cli.add_command(setup_repositories.command, "setup-repositories") cli.add_command(build_containers.command, "build-containers") cli.add_command(build_npms.command, "build-npms") +cli.add_command(build_webapp.command, "build-webapp") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment") diff --git a/stack_orchestrator/opts.py b/stack_orchestrator/opts.py new file mode 100644 index 00000000..665da535 --- /dev/null +++ b/stack_orchestrator/opts.py @@ -0,0 +1,20 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from stack_orchestrator.command_types import CommandOptions + + +class opts: + o: CommandOptions = None diff --git a/stack_orchestrator/repos/__init__.py b/stack_orchestrator/repos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py similarity index 98% rename from app/setup_repositories.py rename to stack_orchestrator/repos/setup_repositories.py index c14dccb4..feca7897 100644 --- a/app/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -25,7 +25,7 @@ import click import importlib.resources from pathlib import Path import yaml -from app.util import include_exclude_check +from stack_orchestrator.util import include_exclude_check class GitProgress(git.RemoteProgress): @@ -232,7 +232,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches os.makedirs(dev_root_path) # See: https://stackoverflow.com/a/20885799/1701505 - from app import data + from stack_orchestrator import data with importlib.resources.open_text(data, "repository-list.txt") as repository_list_file: all_repos = repository_list_file.read().splitlines() @@ -240,7 +240,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches if stack: # In order to be compatible with Python 3.8 we need to use this hack to get the path: # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml") + stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml") with stack_file_path: stack_config = yaml.safe_load(open(stack_file_path, "r")) # TODO: syntax check the input here diff --git a/app/update.py b/stack_orchestrator/update.py similarity index 98% rename from app/update.py rename to stack_orchestrator/update.py index 9f70b06e..a41eabae 100644 --- a/app/update.py +++ b/stack_orchestrator/update.py @@ -23,7 +23,7 @@ import sys import stat import shutil import validators -from app.util import get_yaml +from stack_orchestrator.util import get_yaml def _download_url(url: str, file_path: Path): diff --git a/app/util.py b/stack_orchestrator/util.py similarity index 94% rename from app/util.py rename to stack_orchestrator/util.py index a25aacdb..d3b733a2 100644 --- a/app/util.py +++ b/stack_orchestrator/util.py @@ -79,16 +79,16 @@ def get_pod_list(parsed_stack): return result -def get_plugin_code_path(stack): +def get_plugin_code_paths(stack): parsed_stack = get_parsed_stack_config(stack) pods = parsed_stack["pods"] - # TODO: Hack - pod = pods[0] - if type(pod) is str: - result = get_stack_file_path(stack).parent - else: - pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) - result = Path(os.path.join(pod_root_dir, "stack")) + result = [] + for pod in pods: + if type(pod) is str: + result.append(get_stack_file_path(stack).parent) + else: + pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) + result.append(Path(os.path.join(pod_root_dir, "stack"))) return result diff --git a/app/version.py b/stack_orchestrator/version.py similarity index 96% rename from app/version.py rename to stack_orchestrator/version.py index 5a5c33d4..68e47b44 100644 --- a/app/version.py +++ b/stack_orchestrator/version.py @@ -23,7 +23,7 @@ def command(ctx): '''print tool version''' # See: https://stackoverflow.com/a/20885799/1701505 - from app import data + from stack_orchestrator import data with importlib.resources.open_text(data, "build_tag.txt") as version_file: # TODO: code better version that skips comment lines version_string = version_file.read().splitlines()[1] diff --git a/tests/fixturenet-eth/run-test.sh b/tests/fixturenet-eth/run-test.sh index b5654584..743880e6 100755 --- a/tests/fixturenet-eth/run-test.sh +++ b/tests/fixturenet-eth/run-test.sh @@ -24,13 +24,45 @@ echo "$(date +"%Y-%m-%d %T"): Stack started" $TEST_TARGET_SO --stack fixturenet-eth deploy ps # echo "$(date +"%Y-%m-%d %T"): Getting stack status" # $TEST_TARGET_SO --stack fixturenet-eth deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh -echo "$(date +"%Y-%m-%d %T"): Getting initial block number" -initial_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number") -# Check that the block number increases some time later -sleep 120 -echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number" -subsequent_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number") + +timeout=900 # 15 minutes +echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds" +start_time=$(date +%s) +elapsed_time=0 +initial_block_number=0 +while [ "$initial_block_number" -eq 0 ] && [ $elapsed_time -lt $timeout ]; do + sleep 10 + echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..." + initial_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number") + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) +done + +subsequent_block_number=$initial_block_number + +# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block +if [[ $initial_block_number -gt 0 ]]; then + timeout=300 + echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds" + start_time=$(date +%s) + elapsed_time=0 + # wait for 5 blocks or timeout + while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ] && [ $elapsed_time -lt $timeout ]; do + sleep 10 + echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..." + subsequent_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number") + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + done +fi + +# will return 0 if either of the above loops timed out block_number_difference=$((subsequent_block_number - initial_block_number)) + +echo "$(date +"%Y-%m-%d %T"): Results of block height queries:" +echo "Initial block height: $initial_block_number" +echo "Subsequent block height: $subsequent_block_number" + # Block height difference should be between 1 and some small number if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then echo "Test passed" @@ -41,7 +73,7 @@ else $TEST_TARGET_SO --stack fixturenet-eth deploy logs test_result=1 fi -$TEST_TARGET_SO --stack fixturenet-eth deploy down +$TEST_TARGET_SO --stack fixturenet-eth deploy down --delete-volumes echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories" rm -rf $CERC_REPO_BASE_DIR echo "$(date +"%Y-%m-%d %T"): Test finished" diff --git a/tests/fixturenet-laconicd/run-test.sh b/tests/fixturenet-laconicd/run-test.sh new file mode 100755 index 00000000..8dad9917 --- /dev/null +++ b/tests/fixturenet-laconicd/run-test.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Laconicd fixturenet test" +env +cat /etc/hosts +# Bit of a hack, test the most recent package +TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) +# Set a new unique repo dir +export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX) +echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO" +echo "$(date +"%Y-%m-%d %T"): Test version command" +reported_version_string=$( $TEST_TARGET_SO version ) +echo "$(date +"%Y-%m-%d %T"): Version reported is: ${reported_version_string}" + +echo "$(date +"%Y-%m-%d %T"): Cloning laconicd repositories into: $CERC_REPO_BASE_DIR" +$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories + +echo "$(date +"%Y-%m-%d %T"): Building containers" +$TEST_TARGET_SO --stack fixturenet-laconicd build-containers +echo "$(date +"%Y-%m-%d %T"): Starting stack" +$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd up +echo "$(date +"%Y-%m-%d %T"): Stack started" +# Verify that the fixturenet is up and running +$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps + +timeout=900 # 15 minutes +echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds" +start_time=$(date +%s) +elapsed_time=0 +initial_block_number=null +while [ "$initial_block_number" == "null" ] && [ $elapsed_time -lt $timeout ]; do + sleep 10 + echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..." + initial_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height) + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) +done + +subsequent_block_number=$initial_block_number + +# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block +if [[ $initial_block_number != "null" ]]; then + timeout=300 + echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds" + start_time=$(date +%s) + elapsed_time=0 + # wait for 5 blocks or timeout + while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ] && [ $elapsed_time -lt $timeout ]; do + sleep 10 + echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..." + subsequent_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height) + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + done +fi + +# will return 0 if either of the above loops timed out +block_number_difference=$((subsequent_block_number - initial_block_number)) + +echo "$(date +"%Y-%m-%d %T"): Results of block height queries:" +echo "Initial block height: $initial_block_number" +echo "Subsequent block height: $subsequent_block_number" + +# Block height difference should be between 1 and some small number +if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then + echo "Test passed" + test_result=0 +else + echo "Test failed: block numbers were ${initial_block_number} and ${subsequent_block_number}" + echo "Logs from stack:" + $TEST_TARGET_SO --stack fixturenet-laconicd deploy logs + test_result=1 +fi + +$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd down --delete-volumes +echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories" +rm -rf $CERC_REPO_BASE_DIR +echo "$(date +"%Y-%m-%d %T"): Test finished" +exit $test_result diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh new file mode 100755 index 00000000..91c7890c --- /dev/null +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +# Note: eventually this test should be folded into ../deploy/ +# but keeping it separate for now for convenience +TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) +# Dump environment variables for debugging +echo "Environment variables:" +env +# Set a non-default repo dir +export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir +echo "Testing this package: $TEST_TARGET_SO" +echo "Test version command" +reported_version_string=$( $TEST_TARGET_SO version ) +echo "Version reported is: ${reported_version_string}" +echo "Cloning repositories into: $CERC_REPO_BASE_DIR" +rm -rf $CERC_REPO_BASE_DIR +mkdir -p $CERC_REPO_BASE_DIR +# Test basic stack-orchestrator deploy +test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir +test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml +$TEST_TARGET_SO --stack test deploy --deploy-to k8s init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED +# Check the file now exists +if [ ! -f "$test_deployment_spec" ]; then + echo "deploy init test: spec file not present" + echo "deploy init test: FAILED" + exit 1 +fi +echo "deploy init test: passed" +$TEST_TARGET_SO --stack test deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir +# Check the deployment dir exists +if [ ! -d "$test_deployment_dir" ]; then + echo "deploy create test: deployment directory not present" + echo "deploy create test: FAILED" + exit 1 +fi +echo "deploy create test: passed" +# Check the file writted by the create command in the stack now exists +if [ ! -f "$test_deployment_dir/create-file" ]; then + echo "deploy create test: create output file not present" + echo "deploy create test: FAILED" + exit 1 +fi +# And has the right content +create_file_content=$(<$test_deployment_dir/create-file) +if [ ! "$create_file_content" == "create-command-output-data" ]; then + echo "deploy create test: create output file contents not correct" + echo "deploy create test: FAILED" + exit 1 +fi +echo "deploy create output file test: passed" +# Try to start the deployment +$TEST_TARGET_SO deployment --dir $test_deployment_dir start +# TODO: add a check to see if the container is up +# Sleep because k8s not up yet +sleep 30 +# Check logs command works +log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then + echo "deployment logs test: passed" +else + echo "deployment logs test: FAILED" + exit 1 +fi +# Check the config variable CERC_TEST_PARAM_1 was passed correctly +if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then + echo "deployment config test: passed" +else + echo "deployment config test: FAILED" + exit 1 +fi +# Stop and clean up +$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes +echo "Test passed" diff --git a/tests/webapp-test/run-webapp-test.sh b/tests/webapp-test/run-webapp-test.sh new file mode 100755 index 00000000..71b4da16 --- /dev/null +++ b/tests/webapp-test/run-webapp-test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +# Dump environment variables for debugging +echo "Environment variables:" +env +# Test basic stack-orchestrator webapp +echo "Running stack-orchestrator webapp test" +# Bit of a hack, test the most recent package +TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) +# Set a non-default repo dir +export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir +echo "Testing this package: $TEST_TARGET_SO" +echo "Test version command" +reported_version_string=$( $TEST_TARGET_SO version ) +echo "Version reported is: ${reported_version_string}" +echo "Cloning repositories into: $CERC_REPO_BASE_DIR" +rm -rf $CERC_REPO_BASE_DIR +mkdir -p $CERC_REPO_BASE_DIR +git clone https://git.vdb.to/cerc-io/test-progressive-web-app.git $CERC_REPO_BASE_DIR/test-progressive-web-app + +# Test webapp command execution +$TEST_TARGET_SO build-webapp --source-repo $CERC_REPO_BASE_DIR/test-progressive-web-app + +UUID=`uuidgen` + +set +e + +CONTAINER_ID=$(docker run -p 3000:3000 -d cerc/test-progressive-web-app:local) +sleep 3 +wget -O test.before -m http://localhost:3000 + +docker remove -f $CONTAINER_ID + +CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$UUID -d cerc/test-progressive-web-app:local) +sleep 3 +wget -O test.after -m http://localhost:3000 + +docker remove -f $CONTAINER_ID + +echo "###########################################################################" +echo "" + +grep "$UUID" test.before > /dev/null +if [ $? -ne 1 ]; then + echo "BEFORE: FAILED" + exit 1 +else + echo "BEFORE: PASSED" +fi + +grep "$UUID" test.after > /dev/null +if [ $? -ne 0 ]; then + echo "AFTER: FAILED" + exit 1 +else + echo "AFTER: PASSED" +fi + +exit 0