Compare commits

..
Author SHA1 Message Date
zramsay 1fa67db025 rm old envs 2024-02-19 18:56:44 +00:00
zramsay 3af3915f15 mars v2 2024-02-19 18:56:44 +00:00
72 changed files with 513 additions and 1677 deletions
@@ -1,53 +0,0 @@
name: Fixturenet-Eth-Plugeth-Arm-Test
on:
push:
branches: '*'
paths:
- '!**'
- '.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test'
schedule: # Note: coordinate with other tests to not overload runners at the same time of day
- cron: '2 14 * * *'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
name: "Run an Ethereum plugeth fixturenet test"
runs-on: ubuntu-latest-arm
steps:
- name: "Clone project repository"
uses: actions/checkout@v3
# At present the stock setup-python action fails on Linux/aarch64
# Conditional steps below workaroud this by using deadsnakes for that case only
- name: "Install Python for ARM on Linux"
if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
uses: deadsnakes/action@v3.0.1
with:
python-version: '3.8'
- name: "Install Python cases other than ARM on Linux"
if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: "Run fixturenet-eth tests"
run: ./tests/fixturenet-eth-plugeth/run-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -47,11 +47,3 @@ jobs:
sleep 5
- name: "Run fixturenet-eth tests"
run: ./tests/fixturenet-eth-plugeth/run-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+1 -8
View File
@@ -45,11 +45,4 @@ jobs:
sleep 5
- name: "Run fixturenet-eth tests"
run: ./tests/fixturenet-eth/run-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+1 -11
View File
@@ -11,7 +11,7 @@ on:
jobs:
test:
name: "Run Laconicd fixturenet and Laconic CLI tests"
name: "Run an Laconicd fixturenet test"
runs-on: ubuntu-latest
steps:
- name: 'Update'
@@ -46,13 +46,3 @@ jobs:
run: ./scripts/build_shiv_package.sh
- name: "Run fixturenet-laconicd tests"
run: ./tests/fixturenet-laconicd/run-test.sh
- name: "Run laconic CLI tests"
run: ./tests/fixturenet-laconicd/run-cli-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-8
View File
@@ -19,11 +19,3 @@ jobs:
python-version: '3.8'
- name : "Run flake8"
uses: py-actions/flake8@v2
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-8
View File
@@ -54,11 +54,3 @@ jobs:
# Hack using endsWith to workaround Gitea sometimes sending "publish-test" vs "refs/heads/publish-test"
draft: ${{ endsWith('publish-test', github.ref ) }}
files: ./laconic-so
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+1 -8
View File
@@ -51,11 +51,4 @@ jobs:
source /opt/bash-utils/cgroup-helper.sh
join_cgroup
./tests/container-registry/run-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+1 -8
View File
@@ -49,11 +49,4 @@ jobs:
source /opt/bash-utils/cgroup-helper.sh
join_cgroup
./tests/database/run-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-8
View File
@@ -47,11 +47,3 @@ jobs:
sleep 5
- name: "Run deploy tests"
run: ./tests/deploy/run-deploy-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-8
View File
@@ -51,12 +51,4 @@ jobs:
source /opt/bash-utils/cgroup-helper.sh
join_cgroup
./tests/k8s-deploy/run-deploy-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-8
View File
@@ -49,11 +49,3 @@ jobs:
sleep 5
- name: "Run webapp tests"
run: ./tests/webapp-test/run-webapp-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+2 -8
View File
@@ -47,11 +47,5 @@ jobs:
sleep 5
- name: "Run smoke tests"
run: ./tests/smoke-test/run-smoke-test.sh
- name: Notify Slack on CI failure
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -1,2 +0,0 @@
Change this file to trigger running the fixturenet-eth-plugeth-arm-test CI job
@@ -1,5 +1,3 @@
Change this file to trigger running the fixturenet-laconicd-test CI job
Trigger
Trigger
Trigger
Trigger
+4 -2
View File
@@ -26,7 +26,8 @@ curl -SL https://github.com/docker/compose/releases/download/v2.11.2/docker-comp
chmod +x ~/.docker/cli-plugins/docker-compose
```
Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory.
Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be
a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory.
Now, having selected that directory, download the latest release from [this page](https://git.vdb.to/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
@@ -55,7 +56,6 @@ echo "distribution-url: https://git.vdb.to/cerc-io/stack-orchestrator/releases/d
```
### Update
If Stack Orchestrator was installed using the process described above, it is able to subsequently self-update to the current latest version by running:
```bash
@@ -78,3 +78,5 @@ See the [CONTRIBUTING.md](/docs/CONTRIBUTING.md) for developer mode install.
## Platform Support
Native aarm64 is _not_ currently supported. x64 emulation on ARM64 macos should work (not yet tested).
-9
View File
@@ -1,9 +0,0 @@
# Fetching pre-built container images
When Stack Orchestrator deploys a stack containing a suite of one or more containers it expects images for those containers to be on the local machine with a tag of the form `<image-name>:local` Images for these containers can be built from source (and optionally base container images from public registries) with the `build-containers` subcommand.
However, the task of building a large number of containers from source may consume considerable time and machine resources. This is where the `fetch-containers` subcommand steps in. It is designed to work exactly like `build-containers` but instead the images, pre-built, are fetched from an image registry then re-tagged for deployment. It can be used in place of `build-containers` for any stack provided the necessary containers, built for the local machine architecture (e.g. arm64 or x86-64) have already been published in an image registry.
## Usage
To use `fetch-containers`, provide an image registry path, a username and token/password with read access to the registry, and optionally specify `--force-local-overwrite`. If this argument is not specified, if there is already a locally built or previously fetched image for a stack container on the machine, it will not be overwritten and a warning issued.
```
$ laconic-so --stack mobymask-v3-demo fetch-containers --image-registry git.vdb.to/cerc-io --registry-username <registry-user> --registry-token <registry-token> --force-local-overwrite
```
+97 -104
View File
@@ -15,139 +15,132 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
16 GB Memory / 8 Intel vCPUs / 160 GB Disk.
1. Login to the droplet as root (either by SSH key or password set in the DO console)
```
ssh root@IP
```
1. Get the install script, give it executable permissions, and run it:
```
ssh root@IP
```
```
curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-linux.sh
```
```
chmod +x install.sh
```
```
bash install.sh
```
2. Get the install script, give it executable permissions, and run it:
1. Confirm docker was installed and activate the changes in `~/.profile`:
```
curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-linux.sh
```
```
chmod +x install.sh
```
```
bash install.sh
```
```
docker run hello-world
```
```
source ~/.profile
```
3. Confirm docker was installed and activate the changes in `~/.profile`:
1. Verify installation:
```
docker run hello-world
```
```
source ~/.profile
```
```
laconic-so version
```
4. Verify installation:
```
laconic-so version
```
## Setup the laconic fixturenet stack
1. Get the repositories
```
laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
```
```
laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
```
1. Build the containers:
2. Set this environment variable to the Laconic self-hosted Gitea instance:
```
laconic-so --stack fixturenet-laconic-loaded build-containers
```
```
export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
```
It's possible to run into an `ESOCKETTIMEDOUT` error, e.g., `error An unexpected error occurred: "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz: ESOCKETTIMEDOUT"`. This may happen even if you have a great internet connection. In that case, re-run the `build-containers` command.
3. Build the containers:
```
laconic-so --stack fixturenet-laconic-loaded build-containers
```
1. Set this environment variable to your droplet's IP address or fully qualified DNS host name if it has one:
It's possible to run into an `ESOCKETTIMEDOUT` error, e.g., `error An unexpected error occurred: "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz: ESOCKETTIMEDOUT"`. This may happen even if you have a great internet connection. In that case, re-run the `build-containers` command.
```
export BACKEND_ENDPOINT=http://<your-IP-or-hostname>:9473
```
e.g.
```
export BACKEND_ENDPOINT=http://my-test-server.example.com:9473
```
4. Set this environment variable to your droplet's IP address:
1. Create a deployment directory for the stack:
```
laconic-so --stack fixturenet-laconic-loaded deploy init --output laconic-loaded.spec --map-ports-to-host any-same --config LACONIC_HOSTED_ENDPOINT=$BACKEND_ENDPOINT
```
```
laconic-so --stack fixturenet-laconic-loaded deploy create --deployment-dir laconic-loaded-deployment --spec-file laconic-loaded.spec
```
2. Start the stack:
```
export LACONIC_HOSTED_ENDPOINT=http://<your-IP>
```
```
laconic-so deployment --dir laconic-loaded-deployment start
```
5. Deploy the stack:
3. Check the logs:
```
laconic-so --stack fixturenet-laconic-loaded deploy up
```
```
laconic-so deployment --dir laconic-loaded-deployment logs
```
6. Check the logs:
You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
```
laconic-so --stack fixturenet-laconic-loaded deploy logs
```
```
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:29PM INF indexed block exents height=12 module=txindex server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF indexed block exents height=13 module=txindex server=node
```
You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
4. Confirm operation of the registry CLI:
```
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:29PM INF indexed block exents height=12 module=txindex server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1 | 9:30PM INF indexed block exents height=13 module=txindex server=node
```
```
laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic cns status"
```
7. Confirm operation of the registry CLI:
```
{
"version": "0.3.0",
"node": {
"id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
"network": "laconic_9000-1",
"moniker": "localtestnet"
},
"sync": {
"latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
"latest_block_height": "5",
"latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
"catching_up": false
},
"validator": {
"address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
"voting_power": "1000000000000000"
},
"validators": [
{
"address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
"voting_power": "1000000000000000",
"proposer_priority": "0"
}
],
"num_peers": "0",
"peers": [],
"disk_usage": "292.0K"
}
```
```
laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
```
```
{
"version": "0.3.0",
"node": {
"id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
"network": "laconic_9000-1",
"moniker": "localtestnet"
},
"sync": {
"latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
"latest_block_height": "5",
"latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
"catching_up": false
},
"validator": {
"address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
"voting_power": "1000000000000000"
},
"validators": [
{
"address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
"voting_power": "1000000000000000",
"proposer_priority": "0"
}
],
"num_peers": "0",
"peers": [],
"disk_usage": "292.0K"
}
```
## Configure Digital Ocean firewall
(Note this step may not be necessary depending on the droplet image used)
Let's open some ports.
1. In the Digital Ocean web console, navigate to your droplet's main page. Select the "Networking" tab and scroll down to "Firewall".
+68 -61
View File
@@ -25,13 +25,10 @@ import sys
from decouple import config
import subprocess
import click
import importlib.resources
from pathlib import Path
from stack_orchestrator.opts import opts
from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, warn_exit
from stack_orchestrator.base import get_npm_registry_url
from stack_orchestrator.build.build_types import BuildContext
from stack_orchestrator.build.publish import publish_image
from stack_orchestrator.build.build_util import get_containers_in_scope
# TODO: find a place for this
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -62,58 +59,69 @@ def make_container_build_env(dev_root_path: str,
return container_build_env
def process_container(build_context: BuildContext) -> bool:
if not opts.o.quiet:
print(f"Building: {build_context.container}")
def process_container(stack: str,
container,
container_build_dir: str,
container_build_env: dict,
dev_root_path: str,
quiet: bool,
verbose: bool,
dry_run: bool,
continue_on_error: bool,
):
if not quiet:
print(f"Building: {container}")
default_container_tag = f"{build_context.container}:local"
build_context.container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
default_container_tag = f"{container}:local"
container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
# Check if this is in an external stack
if stack_is_external(build_context.stack):
container_parent_dir = Path(build_context.stack).joinpath("container-build")
temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
if stack_is_external(stack):
container_parent_dir = Path(stack).joinpath("container-build")
temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
temp_build_script_filename = temp_build_dir.joinpath("build.sh")
# Now check if the container exists in the external stack.
if not temp_build_script_filename.exists():
# If not, revert to building an internal container
container_parent_dir = build_context.container_build_dir
container_parent_dir = container_build_dir
else:
container_parent_dir = build_context.container_build_dir
container_parent_dir = container_build_dir
build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
build_script_filename = build_dir.joinpath("build.sh")
if opts.o.verbose:
if verbose:
print(f"Build script filename: {build_script_filename}")
if os.path.exists(build_script_filename):
build_command = build_script_filename.as_posix()
else:
if opts.o.verbose:
if verbose:
print(f"No script file found: {build_script_filename}, using default build script")
repo_dir = build_context.container.split('/')[1]
repo_dir = container.split('/')[1]
# TODO: make this less of a hack -- should be specified in some metadata somewhere
# Check if we have a repo for this container. If not, set the context dir to the container-build subdir
repo_full_path = os.path.join(build_context.dev_root_path, repo_dir)
repo_full_path = os.path.join(dev_root_path, repo_dir)
repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
build_command = os.path.join(build_context.container_build_dir,
build_command = os.path.join(container_build_dir,
"default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
if not opts.o.dry_run:
if not dry_run:
# No PATH at all causes failures with podman.
if "PATH" not in build_context.container_build_env:
build_context.container_build_env["PATH"] = os.environ["PATH"]
if opts.o.verbose:
print(f"Executing: {build_command} with environment: {build_context.container_build_env}")
build_result = subprocess.run(build_command, shell=True, env=build_context.container_build_env)
if opts.o.verbose:
if "PATH" not in container_build_env:
container_build_env["PATH"] = os.environ["PATH"]
if verbose:
print(f"Executing: {build_command} with environment: {container_build_env}")
build_result = subprocess.run(build_command, shell=True, env=container_build_env)
if verbose:
print(f"Return code is: {build_result.returncode}")
if build_result.returncode != 0:
return False
else:
return True
print(f"Error running build for {container}")
if not continue_on_error:
print("FATAL Error: container build failed and --continue-on-error not set, exiting")
sys.exit(1)
else:
print("****** Container Build Error, continuing because --continue-on-error is set")
else:
print("Skipped")
return True
@click.command()
@@ -121,14 +129,17 @@ def process_container(build_context: BuildContext) -> bool:
@click.option('--exclude', help="don\'t build these containers")
@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
@click.option("--extra-build-args", help="Supply extra arguments to build")
@click.option("--publish-images", is_flag=True, default=False, help="Publish the built images in the specified image registry")
@click.option("--image-registry", help="Specify the image registry for --publish-images")
@click.pass_context
def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_images, image_registry):
def command(ctx, include, exclude, force_rebuild, extra_build_args):
'''build the set of containers required for a complete stack'''
quiet = ctx.obj.quiet
verbose = ctx.obj.verbose
dry_run = ctx.obj.dry_run
debug = ctx.obj.debug
local_stack = ctx.obj.local_stack
stack = ctx.obj.stack
continue_on_error = ctx.obj.continue_on_error
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
@@ -139,45 +150,41 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
else:
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
if not opts.o.quiet:
if not quiet:
print(f'Dev Root is: {dev_root_path}')
if not os.path.isdir(dev_root_path):
print('Dev root directory doesn\'t exist, creating')
if publish_images:
if not image_registry:
error_exit("--image-registry must be supplied with --publish-images")
# See: https://stackoverflow.com/a/20885799/1701505
from stack_orchestrator import data
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
all_containers = container_list_file.read().splitlines()
containers_in_scope = get_containers_in_scope(stack)
containers_in_scope = []
if stack:
stack_config = get_parsed_stack_config(stack)
if "containers" not in stack_config or stack_config["containers"] is None:
warn_exit(f"stack {stack} does not define any containers")
containers_in_scope = stack_config['containers']
else:
containers_in_scope = all_containers
if verbose:
print(f'Containers: {containers_in_scope}')
if stack:
print(f"Stack: {stack}")
container_build_env = make_container_build_env(dev_root_path,
container_build_dir,
opts.o.debug,
debug,
force_rebuild,
extra_build_args)
for container in containers_in_scope:
if include_exclude_check(container, include, exclude):
build_context = BuildContext(
stack,
container,
container_build_dir,
container_build_env,
dev_root_path
)
result = process_container(build_context)
if result:
if publish_images:
publish_image(f"{container}:local", image_registry)
else:
print(f"Error running build for {build_context.container}")
if not opts.o.continue_on_error:
error_exit("container build failed and --continue-on-error not set, exiting")
sys.exit(1)
else:
print("****** Container Build Error, continuing because --continue-on-error is set")
process_container(stack, container, container_build_dir, container_build_env,
dev_root_path, quiet, verbose, dry_run, continue_on_error)
else:
if opts.o.verbose:
if verbose:
print(f"Excluding: {container}")
-29
View File
@@ -1,29 +0,0 @@
# Copyright © 2024 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from dataclasses import dataclass
from pathlib import Path
from typing import Mapping
@dataclass
class BuildContext:
stack: str
container: str
container_build_dir: Path
container_build_env: Mapping[str,str]
dev_root_path: str
-43
View File
@@ -1,43 +0,0 @@
# Copyright © 2024 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import importlib.resources
from stack_orchestrator.opts import opts
from stack_orchestrator.util import get_parsed_stack_config, warn_exit
def get_containers_in_scope(stack: str):
# See: https://stackoverflow.com/a/20885799/1701505
from stack_orchestrator import data
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
all_containers = container_list_file.read().splitlines()
containers_in_scope = []
if stack:
stack_config = get_parsed_stack_config(stack)
if "containers" not in stack_config or stack_config["containers"] is None:
warn_exit(f"stack {stack} does not define any containers")
containers_in_scope = stack_config['containers']
else:
containers_in_scope = all_containers
if opts.o.verbose:
print(f'Containers: {containers_in_scope}')
if stack:
print(f"Stack: {stack}")
return containers_in_scope
+14 -47
View File
@@ -21,14 +21,11 @@
# TODO: display the available list of containers; allow re-build of either all or specific containers
import os
import sys
from decouple import config
import click
from pathlib import Path
from stack_orchestrator.build import build_containers
from stack_orchestrator.deploy.webapp.util import determine_base_container, TimedLogger
from stack_orchestrator.build.build_types import BuildContext
from stack_orchestrator.deploy.webapp.util import determine_base_container
@click.command()
@@ -40,25 +37,26 @@ from stack_orchestrator.build.build_types import BuildContext
@click.pass_context
def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
'''build the specified webapp container'''
logger = TimedLogger()
quiet = ctx.obj.quiet
debug = ctx.obj.debug
verbose = ctx.obj.verbose
dry_run = ctx.obj.dry_run
debug = ctx.obj.debug
local_stack = ctx.obj.local_stack
stack = ctx.obj.stack
continue_on_error = ctx.obj.continue_on_error
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
if local_stack:
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
logger.log(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
else:
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
if verbose:
logger.log(f'Dev Root is: {dev_root_path}')
if not quiet:
print(f'Dev Root is: {dev_root_path}')
if not base_container:
base_container = determine_base_container(source_repo)
@@ -67,23 +65,8 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
force_rebuild, extra_build_args)
if verbose:
logger.log(f"Building base container: {base_container}")
build_context_1 = BuildContext(
stack,
base_container,
container_build_dir,
container_build_env,
dev_root_path,
)
ok = build_containers.process_container(build_context_1)
if not ok:
logger.log("ERROR: Build failed.")
sys.exit(1)
if verbose:
logger.log(f"Base container {base_container} build finished.")
build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
verbose, dry_run, continue_on_error)
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
@@ -93,25 +76,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
"Dockerfile.webapp")
if not tag:
webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
tag = f"cerc/{webapp_name}:local"
container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
else:
container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
if verbose:
logger.log(f"Building app container: {tag}")
build_context_2 = BuildContext(
stack,
base_container,
container_build_dir,
container_build_env,
dev_root_path,
)
ok = build_containers.process_container(build_context_2)
if not ok:
logger.log("ERROR: Build failed.")
sys.exit(1)
if verbose:
logger.log(f"App container {base_container} build finished.")
logger.log("build-webapp complete", show_step_time=False, show_total_time=True)
build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
verbose, dry_run, continue_on_error)
@@ -1,195 +0,0 @@
# Copyright © 2024 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import click
from dataclasses import dataclass
import json
import platform
from python_on_whales import DockerClient
from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList
from python_on_whales.utils import run
import requests
from typing import List
from stack_orchestrator.opts import opts
from stack_orchestrator.util import include_exclude_check, error_exit
from stack_orchestrator.build.build_util import get_containers_in_scope
# Experimental fetch-container command
@dataclass
class RegistryInfo:
registry: str
registry_username: str
registry_token: str
# Extending this code to support the --verbose option, cnosider contributing upstream
# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129
class ExtendedManifestCLI(ManifestCLI):
def inspect_verbose(self, x: str) -> ManifestList:
"""Returns a Docker manifest list object."""
json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x])
return json.loads(json_str)
def _local_tag_for(container: str):
return f"{container}:local"
# See: https://docker-docs.uclv.cu/registry/spec/api/
# Emulate this:
# $ curl -u "my-username:my-token" -X GET "https://<container-registry-hostname>/v2/cerc-io/cerc/test-container/tags/list"
# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
# registry looks like: git.vdb.to/cerc-io
registry_parts = registry_info.registry.split("/")
url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
if opts.o.debug:
print(f"Fetching tags from: {url}")
response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token))
if response.status_code == 200:
tag_info = response.json()
if opts.o.debug:
print(f"container tags list: {tag_info}")
tags_array = tag_info["tags"]
return tags_array
else:
error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
def _find_latest(candidate_tags: List[str]):
# Lex sort should give us the latest first
sorted_candidates = sorted(candidate_tags)
if opts.o.debug:
print(f"sorted candidates: {sorted_candidates}")
return sorted_candidates[-1]
def _filter_for_platform(container: str,
registry_info: RegistryInfo,
tag_list: List[str]) -> List[str] :
filtered_tags = []
this_machine = platform.machine()
# Translate between Python and docker platform names
if this_machine == "x86_64":
this_machine = "amd64"
if this_machine == "aarch64":
this_machine = "arm64"
if opts.o.debug:
print(f"Python says the architecture is: {this_machine}")
docker = DockerClient()
for tag in tag_list:
remote_tag = f"{registry_info.registry}/{container}:{tag}"
manifest_cmd = ExtendedManifestCLI(docker.client_config)
manifest = manifest_cmd.inspect_verbose(remote_tag)
if opts.o.debug:
print(f"manifest: {manifest}")
image_architecture = manifest["Descriptor"]["platform"]["architecture"]
if opts.o.debug:
print(f"image_architecture: {image_architecture}")
if this_machine == image_architecture:
filtered_tags.append(tag)
if opts.o.debug:
print(f"Tags filtered for platform: {filtered_tags}")
return filtered_tags
def _get_latest_image(container: str, registry_info: RegistryInfo):
all_tags = _get_tags_for_container(container, registry_info)
tags_for_platform = _filter_for_platform(container, registry_info, all_tags)
if len(tags_for_platform) > 0:
latest_tag = _find_latest(tags_for_platform)
return f"{container}:{latest_tag}"
else:
return None
def _fetch_image(tag: str, registry_info: RegistryInfo):
docker = DockerClient()
remote_tag = f"{registry_info.registry}/{tag}"
if opts.o.debug:
print(f"Attempting to pull this image: {remote_tag}")
docker.image.pull(remote_tag)
def _exists_locally(container: str):
docker = DockerClient()
return docker.image.exists(_local_tag_for(container))
def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
docker = DockerClient()
docker.image.tag(f"{registry}/{remote_tag}", local_tag)
@click.command()
@click.option('--include', help="only fetch these containers")
@click.option('--exclude', help="don\'t fetch these containers")
@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
@click.option("--registry-username", required=True, help="Specify the image registry username")
@click.option("--registry-token", required=True, help="Specify the image registry access token")
@click.pass_context
def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token):
'''EXPERIMENTAL: fetch the images for a stack from remote registry'''
registry_info = RegistryInfo(image_registry, registry_username, registry_token)
docker = DockerClient()
if not opts.o.quiet:
print("Logging into container registry:")
docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
# Generate list of target containers
stack = ctx.obj.stack
containers_in_scope = get_containers_in_scope(stack)
all_containers_found = True
for container in containers_in_scope:
local_tag = _local_tag_for(container)
if include_exclude_check(container, include, exclude):
if opts.o.debug:
print(f"Processing: {container}")
# For each container, attempt to find the latest of a set of
# images with the correct name and platform in the specified registry
image_to_fetch = _get_latest_image(container, registry_info)
if not image_to_fetch:
print(f"Warning: no image found to fetch for container: {container}")
all_containers_found = False
continue
if opts.o.debug:
print(f"Fetching: {image_to_fetch}")
_fetch_image(image_to_fetch, registry_info)
# Now check if the target container already exists exists locally already
if (_exists_locally(container)):
if not opts.o.quiet:
print(f"Container image {container} already exists locally")
# if so, fail unless the user specified force-local-overwrite
if (force_local_overwrite):
# In that case remove the existing :local tag
if not opts.o.quiet:
print(f"Warning: overwriting local tag from this image: {container} because "
"--force-local-overwrite was specified")
else:
if not opts.o.quiet:
print(f"Skipping local tagging for this image: {container} because that would "
"overwrite an existing :local tagged image, use --force-local-overwrite to do so.")
continue
# Tag the fetched image with the :local tag
_add_local_tag(image_to_fetch, image_registry, local_tag)
else:
if opts.o.verbose:
print(f"Excluding: {container}")
if not all_containers_found:
print("Warning: couldn't find usable images for one or more containers, this stack will not deploy")
-48
View File
@@ -1,48 +0,0 @@
# Copyright © 2024 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from datetime import datetime
from python_on_whales import DockerClient
from stack_orchestrator.opts import opts
from stack_orchestrator.util import error_exit
def _publish_tag_for_image(local_image_tag: str, remote_repo: str, version: str):
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
(image_name, image_version) = local_image_tag.split(":")
if image_version == "local":
return f"{remote_repo}/{image_name}:{version}"
else:
error_exit("Asked to publish a non-locally built image")
def publish_image(local_tag, registry):
if opts.o.verbose:
print(f"Publishing this image: {local_tag} to this registry: {registry}")
docker = DockerClient()
# Figure out the target image tag
# Eventually this version will be generated from the source repo state
# Using a timestemp is an intermediate step
version = datetime.now().strftime("%Y%m%d%H%M")
remote_tag = _publish_tag_for_image(local_tag, registry, version)
# Tag the image thus
if opts.o.debug:
print(f"Tagging {local_tag} to {remote_tag}")
docker.image.tag(local_tag, remote_tag)
# Push it to the desired registry
if opts.o.verbose:
print(f"Pushing image {remote_tag}")
docker.image.push(remote_tag)
@@ -5,11 +5,9 @@ services:
environment:
REGISTRY_LOG_LEVEL: ${REGISTRY_LOG_LEVEL}
volumes:
- config:/config:ro
- registry-data:/var/lib/registry
ports:
- "5000"
volumes:
config:
registry-data:
@@ -3,9 +3,6 @@ services:
restart: unless-stopped
image: cerc/laconicd:local
command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
environment:
TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED}
TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY}
volumes:
# The cosmos-sdk node's database directory:
- laconicd-data:/root/.laconicd
@@ -28,7 +25,6 @@ services:
image: cerc/laconic-registry-cli:local
volumes:
- ../config/fixturenet-laconicd/registry-cli-config-template.yml:/registry-cli-config-template.yml
- ${BASE_DIR:-~/cerc}/laconic-registry-cli:/laconic-registry-cli
volumes:
laconicd-data:
@@ -1,13 +0,0 @@
services:
snowballtools-base-backend:
image: cerc/snowballtools-base-backend:local
restart: always
volumes:
- data:/data
- config:/config:ro
ports:
- 8000
volumes:
data:
config:
@@ -6,7 +6,6 @@ services:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED}
CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE"
CERC_TEST_PARAM_3: ${CERC_TEST_PARAM_3:-FAILED}
volumes:
- test-data-bind:/data
- test-data-auto:/data2
@@ -1,76 +0,0 @@
version: '3.2'
services:
ajna-watcher-db:
restart: unless-stopped
image: postgres:14-alpine
environment:
- POSTGRES_USER=vdbm
- POSTGRES_MULTIPLE_DATABASES=ajna-watcher,ajna-watcher-job-queue
- POSTGRES_EXTENSION=ajna-watcher-job-queue:pgcrypto
- POSTGRES_PASSWORD=password
volumes:
- ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
- ajna_watcher_db_data:/var/lib/postgresql/data
ports:
- "5432"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "5432"]
interval: 20s
timeout: 5s
retries: 15
start_period: 10s
ajna-watcher-job-runner:
restart: unless-stopped
depends_on:
ajna-watcher-db:
condition: service_healthy
image: cerc/watcher-ajna:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-job-runner.sh"]
volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
- ../config/watcher-ajna/start-job-runner.sh:/app/start-job-runner.sh
ports:
- "9000"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "9000"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
ajna-watcher-server:
restart: unless-stopped
depends_on:
ajna-watcher-db:
condition: service_healthy
ajna-watcher-job-runner:
condition: service_healthy
image: cerc/watcher-ajna:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-server.sh"]
volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
- ../config/watcher-ajna/start-server.sh:/app/start-server.sh
ports:
- "3008"
- "9001"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "3008"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
ajna_watcher_db_data:
@@ -50,6 +50,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -126,6 +142,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -202,6 +234,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -278,6 +326,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -354,6 +418,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -430,6 +510,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -506,6 +602,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -582,6 +694,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -660,6 +788,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -736,6 +880,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -1,20 +0,0 @@
#!/bin/sh
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
set -u
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
echo "Running job-runner..."
DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js
@@ -1,20 +0,0 @@
#!/bin/sh
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
set -u
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
echo "Running server..."
DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js
@@ -1,98 +0,0 @@
[server]
host = "0.0.0.0"
port = 3008
kind = "active"
gqlPath = "/"
# Checkpointing state.
checkpointing = true
# Checkpoint interval in number of blocks.
checkpointInterval = 2000
# Enable state creation
# CAUTION: Disable only if state creation is not desired or can be filled subsequently
enableState = false
subgraphPath = "./subgraph-build"
# Interval to restart wasm instance periodically
wasmRestartBlocksInterval = 20
# Interval in number of blocks at which to clear entities cache.
clearEntitiesCacheInterval = 1000
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Flag to specify whether RPC endpoint supports block hash as block tag parameter
rpcSupportsBlockHashParam = false
# GQL cache settings
[server.gqlCache]
enabled = true
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
[metrics]
host = "0.0.0.0"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"
host = "ajna-watcher-db"
port = 5432
database = "ajna-watcher"
username = "vdbm"
password = "password"
synchronize = true
logging = false
[upstream]
[upstream.ethServer]
rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
# Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
rpcClient = true
# Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
isFEVM = true
# Boolean flag to filter event logs by contracts
filterLogsByAddresses = true
# Boolean flag to filter event logs by topics
filterLogsByTopics = true
[upstream.cache]
name = "requests"
enabled = false
deleteOnStart = false
[jobQueue]
dbConnectionString = "postgres://vdbm:password@ajna-watcher-db/ajna-watcher-job-queue"
maxCompletionLagInSecs = 300
jobDelayInMilliSecs = 100
eventsInBatch = 50
subgraphEventsOrder = true
# Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
blockDelayInMilliSecs = 30000
# Boolean to switch between modes of processing events when starting the server.
# Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
# Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
useBlockRanges = true
# Block range in which logs are fetched during historical blocks processing
historicalLogsBlockRange = 2000
# Max block range of historical processing after which it waits for completion of events processing
# If set to -1 historical processing does not wait for events processing and completes till latest canonical block
historicalMaxFetchAhead = 10000
@@ -26,8 +26,6 @@ RUN \
&& su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
# Install semver
&& su ${USERNAME} -c "umask 0002 && npm install -g semver" \
# Install pnpm
&& su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
&& npm cache clean --force > /dev/null 2>&1
# [Optional] Uncomment this section to install additional OS packages.
@@ -37,9 +35,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# [Optional] Uncomment if you want to install more global node modules
# RUN su node -c "npm install -g <your-package-list-here>"
# We do this to get a yq binary from the published container, for the correct architecture we're building here
COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
# Expose port for http
EXPOSE 80
@@ -11,14 +11,8 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}
docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
rc=$?
if [ $rc -ne 0 ]; then
echo "BUILD FAILED" 1>&2
exit $rc
fi
if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
cat <<EOF
#################################################################
@@ -10,12 +10,10 @@ TRG_DIR="${3:-.next-r}"
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
if [ -f "pnpm-lock.yaml" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
else
if [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=npm
else
CERC_BUILD_TOOL=yarn
fi
fi
@@ -9,9 +9,7 @@ CERC_MIN_NEXTVER=13.4.2
CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
if [ -f "pnpm-lock.yaml" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
if [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
else
CERC_BUILD_TOOL=npm
@@ -115,7 +113,7 @@ if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VE
mv package.json.$$ package.json
fi
time $CERC_BUILD_TOOL install || exit 1
$CERC_BUILD_TOOL install || exit 1
CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json`
@@ -138,9 +136,9 @@ to use for the build with:
EOF
cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$
mv package.json.$$ package.json
time $CERC_BUILD_TOOL install || exit 1
$CERC_BUILD_TOOL install || exit 1
fi
time $CERC_BUILD_TOOL run cerc_compile || exit 1
$CERC_BUILD_TOOL run cerc_compile || exit 1
exit 0
@@ -16,9 +16,7 @@ trap ctrl_c INT
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
if [ -f "pnpm-lock.yaml" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
CERC_BUILD_TOOL=yarn
else
CERC_BUILD_TOOL=npm
@@ -1,6 +0,0 @@
FROM cerc/snowballtools-base-backend-base:local
WORKDIR /app/packages/backend
COPY run.sh .
ENTRYPOINT ["./run.sh"]
@@ -1,26 +0,0 @@
FROM ubuntu:22.04 as builder
RUN apt update && \
apt install -y --no-install-recommends --no-install-suggests \
ca-certificates curl gnupg
# Node
ARG NODE_MAJOR=20
RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
apt update && apt install -y nodejs
# npm setup
RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
COPY . /app/
WORKDIR /app/
RUN find . -name 'node_modules' | xargs -n1 rm -rf
RUN yarn && yarn build --ignore frontend
FROM cerc/webapp-base:local
COPY --from=builder /app /app
WORKDIR /app/packages/backend
@@ -1,10 +0,0 @@
#!/usr/bin/env bash
# Build cerc/webapp-deployer-backend
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
# See: https://stackoverflow.com/a/246128/1701505
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR}
@@ -1,19 +0,0 @@
#!/bin/bash
LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
if [ -f "/config/laconic-hosted-config.yml" ]; then
LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
elif [ -f "/config/config.yml" ]; then
LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
fi
fi
if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
/scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
fi
/scripts/apply-runtime-env.sh "`pwd`/dist"
yarn start
@@ -39,15 +39,6 @@ fi
if [ -n "$CERC_TEST_PARAM_2" ]; then
echo "Test-param-2: ${CERC_TEST_PARAM_2}"
fi
if [ -n "$CERC_TEST_PARAM_3" ]; then
echo "Test-param-3: ${CERC_TEST_PARAM_3}"
fi
if [ -n "$CERC_TEST_PARAM_4" ]; then
echo "Test-param-4: ${CERC_TEST_PARAM_4}"
fi
if [ -n "$CERC_TEST_PARAM_5" ]; then
echo "Test-param-5: ${CERC_TEST_PARAM_5}"
fi
if [ -d "/config" ]; then
echo "/config: EXISTS"
@@ -1,10 +0,0 @@
FROM node:18.17.1-alpine3.18
RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
WORKDIR /app
COPY . .
RUN echo "Installing dependencies and building ajna-watcher-ts" && \
yarn && yarn build
@@ -1,9 +0,0 @@
#!/usr/bin/env bash
# Build cerc/watcher-ajna
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
# See: https://stackoverflow.com/a/246128/1701505
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/watcher-ajna:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/ajna-watcher-ts
@@ -1,6 +1,6 @@
# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
ARG VARIANT=20-bullseye-slim
ARG VARIANT=20-bullseye
FROM node:${VARIANT}
ARG USERNAME=node
@@ -24,10 +24,6 @@ RUN \
&& su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
# Install eslint
&& su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
# Install semver
&& su ${USERNAME} -c "umask 0002 && npm install -g semver" \
# Install pnpm
&& su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
&& npm cache clean --force > /dev/null 2>&1
# [Optional] Uncomment this section to install additional OS packages.
@@ -47,6 +43,7 @@ COPY scripts /scripts
# RUN su node -c "npm install -g <your-package-list-here>"
RUN mkdir -p /config
COPY ./config.yml /config
# Install simple web server for now (use nginx perhaps later)
RUN yarn global add http-server
@@ -1,12 +1,11 @@
FROM cerc/webapp-base:local as builder
ARG CERC_BUILD_TOOL
ARG CERC_BUILD_OUTPUT_DIR
WORKDIR /app
COPY . .
RUN rm -rf node_modules build dist .next*
RUN /scripts/build-app.sh /app /data
RUN rm -rf node_modules build .next*
RUN /scripts/build-app.sh /app build /data
FROM cerc/webapp-base:local
COPY --from=builder /data /data
@@ -11,14 +11,8 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local}
docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
rc=$?
if [ $rc -ne 0 ]; then
echo "BUILD FAILED" 1>&2
exit $rc
fi
if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
cat <<EOF
#################################################################
@@ -0,0 +1 @@
# Put config here.
@@ -7,46 +7,27 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
CERC_BUILD_OUTPUT_DIR="${CERC_BUILD_OUTPUT_DIR}"
WORK_DIR="${1:-/app}"
DEST_DIR="${2:-/data}"
OUTPUT_DIR="${2:-build}"
DEST_DIR="${3:-/data}"
if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
cd "${WORK_DIR}" || exit 1
rm -rf "${DEST_DIR}"
./build-webapp.sh "${DEST_DIR}" || exit 1
elif [ -f "${WORK_DIR}/package.json" ]; then
if [ -f "${WORK_DIR}/package.json" ]; then
echo "Building node-based webapp ..."
cd "${WORK_DIR}" || exit 1
if [ -z "$CERC_BUILD_TOOL" ]; then
if [ -f "pnpm-lock.yaml" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
if [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
else
CERC_BUILD_TOOL=npm
fi
fi
time $CERC_BUILD_TOOL install || exit 1
time $CERC_BUILD_TOOL build || exit 1
$CERC_BUILD_TOOL install || exit 1
$CERC_BUILD_TOOL build || exit 1
rm -rf "${DEST_DIR}"
if [ -z "${CERC_BUILD_OUTPUT_DIR}" ]; then
if [ -d "${WORK_DIR}/dist" ]; then
CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/dist"
elif [ -d "${WORK_DIR}/build" ]; then
CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/build"
else
echo "ERROR: Unable to locate build output. Set with --extra-build-args \"--build-arg CERC_BUILD_OUTPUT_DIR=path\"" 1>&2
exit 1
fi
fi
mv "${CERC_BUILD_OUTPUT_DIR}" "${DEST_DIR}"
mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
else
echo "Copying static app ..."
mv "${WORK_DIR}" "${DEST_DIR}"
@@ -3,40 +3,13 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
CERC_LISTEN_PORT=${CERC_LISTEN_PORT:-80}
CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}"
if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then
if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then
CERC_SINGLE_PAGE_APP=true
else
CERC_SINGLE_PAGE_APP=false
fi
fi
if [ "true" == "$CERC_ENABLE_CORS" ]; then
CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
fi
if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then
# Create a catchall redirect back to /
CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?"
fi
LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
if [ -f "/config/laconic-hosted-config.yml" ]; then
LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
elif [ -f "/config/config.yml" ]; then
LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
fi
fi
if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
/scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "${CERC_WEBAPP_FILES_DIR}"
fi
/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR}
/scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR}
@@ -1,118 +0,0 @@
# Ajna Watcher
## Setup
Clone required repositories:
```bash
laconic-so --stack ajna setup-repositories --git-ssh --pull
```
Build the container images:
```bash
laconic-so --stack ajna build-containers
```
## Deploy
Create a spec file for the deployment:
```bash
laconic-so --stack ajna deploy init --output ajna-spec.yml
```
### Ports
Edit `network` in the spec file to map container ports to host ports as required:
```yml
...
network:
ports:
ajna-watcher-db:
- 15432:5432
ajna-watcher-job-runner:
- 9000:9000
ajna-watcher-server:
- 3008:3008
- 9001:9001
```
### Create a deployment
Create a deployment from the spec file:
```bash
laconic-so --stack ajna deploy create --spec-file ajna-spec.yml --deployment-dir ajna-deployment
```
### Configuration
Inside deployment directory, open the `config.env` file and set following env variables:
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
```bash
laconic-so deployment --dir ajna-deployment start
```
* To list down and monitor the running containers:
```bash
# With status
docker ps -a
# Check logs for a container
docker logs -f <CONTAINER_ID>
```
* Open the GQL playground at <http://localhost:3008/graphql>
```graphql
# Example query
query {
_meta {
block {
hash
number
timestamp
}
deployment
hasIndexingErrors
}
accounts {
id
txCount
tokensDelegated
rewardsClaimed
}
}
```
## Clean up
Stop all the ajna services running in background:
```bash
# Only stop the docker containers
laconic-so deployment --dir ajna-deployment stop
# Run 'start' to restart the deployment
```
To stop all the ajna services and also delete data:
```bash
# Stop the docker containers
laconic-so deployment --dir ajna-deployment stop --delete-volumes
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r ajna-deployment
```
@@ -1,9 +0,0 @@
version: "1.0"
name: ajna
description: "Ajna watcher stack"
repos:
- git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
containers:
- cerc/watcher-ajna
pods:
- watcher-ajna
@@ -16,55 +16,26 @@ laconic-so --stack merkl-sushiswap-v3 build-containers
## Deploy
Create a spec file for the deployment:
```bash
laconic-so --stack merkl-sushiswap-v3 deploy init --output merkl-sushiswap-v3-spec.yml
```
### Ports
Edit `network` in the spec file to map container ports to host ports as required:
```
...
network:
ports:
merkl-sushiswap-v3-watcher-db:
- '5432'
merkl-sushiswap-v3-watcher-job-runner:
- 9002:9000
merkl-sushiswap-v3-watcher-server:
- 127.0.0.1:3007:3008
- 9003:9001
```
### Create a deployment
Create a deployment from the spec file:
```bash
laconic-so --stack merkl-sushiswap-v3 deploy create --spec-file merkl-sushiswap-v3-spec.yml --deployment-dir merkl-sushiswap-v3-deployment
```
### Configuration
Inside deployment directory, open the `config.env` file and set following env variables:
Create and update an env file to be used in the next step:
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher
CERC_ETH_RPC_ENDPOINT=
```
### Deploy the stack
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
```bash
laconic-so deployment --dir merkl-sushiswap-v3-deployment start
laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
```
* To list down and monitor the running containers:
```bash
laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps
# With status
docker ps -a
@@ -75,7 +46,6 @@ laconic-so deployment --dir merkl-sushiswap-v3-deployment start
* Open the GQL playground at http://localhost:3007/graphql
```graphql
# Example query
{
_meta {
block {
@@ -84,7 +54,7 @@ laconic-so deployment --dir merkl-sushiswap-v3-deployment start
}
hasIndexingErrors
}
factories {
id
poolCount
@@ -94,21 +64,18 @@ laconic-so deployment --dir merkl-sushiswap-v3-deployment start
## Clean up
Stop all the merkl-sushiswap-v3 services running in background:
Stop all the services running in background:
```bash
# Only stop the docker containers
laconic-so deployment --dir merkl-sushiswap-v3-deployment stop
# Run 'start' to restart the deployment
laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down
```
To stop all the merkl-sushiswap-v3 services and also delete data:
Clear volumes created by this stack:
```bash
# Stop the docker containers
laconic-so deployment --dir merkl-sushiswap-v3-deployment stop --delete-volumes
# List all relevant volumes
docker volume ls -q --filter "name=merkl_sushiswap_v3"
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r merkl-sushiswap-v3-deployment
# Remove all the listed volumes
docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3")
```
@@ -2,7 +2,7 @@ version: "1.0"
name: merkl-sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6
containers:
- cerc/watcher-merkl-sushiswap-v3
pods:
@@ -1,7 +1,7 @@
version: "0.1"
name: monitoring
repos:
- github.com/cerc-io/watcher-ts@v0.2.81
- github.com/cerc-io/watcher-ts@v0.2.79
containers:
- cerc/watcher-ts
pods:
@@ -1,10 +0,0 @@
version: "1.0"
name: snowballtools-base-backend
description: "snowballtools-base-backend"
repos:
- github.com/snowball-tools/snowballtools-base
containers:
- cerc/webapp-base
- cerc/snowballtools-base-backend
pods:
- snowballtools-base-backend
@@ -55,7 +55,7 @@ ports:
Create deployment:
```bash
laconic-so --stack sushiswap-subgraph deploy create --spec-file sushiswap-subgraph-spec.yml --deployment-dir sushiswap-subgraph-deployment
laconic-so deploy create --spec-file sushiswap-subgraph-spec.yml --deployment-dir sushiswap-subgraph-deployment
```
## Start the stack
@@ -16,55 +16,26 @@ laconic-so --stack sushiswap-v3 build-containers
## Deploy
Create a spec file for the deployment:
```bash
laconic-so --stack sushiswap-v3 deploy init --output sushiswap-v3-spec.yml
```
### Ports
Edit `network` in the spec file to map container ports to host ports as required:
```
...
network:
ports:
sushiswap-v3-watcher-db:
- '5432'
sushiswap-v3-watcher-job-runner:
- 9000:9000
sushiswap-v3-watcher-server:
- 127.0.0.1:3008:3008
- 9001:9001
```
### Create a deployment
Create a deployment from the spec file:
```bash
laconic-so --stack sushiswap-v3 deploy create --spec-file sushiswap-v3-spec.yml --deployment-dir sushiswap-v3-deployment
```
### Configuration
Inside deployment directory, open the `config.env` file and set following env variables:
Create and update an env file to be used in the next step:
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher
CERC_ETH_RPC_ENDPOINT=
```
### Deploy the stack
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
```bash
laconic-so deployment --dir sushiswap-v3-deployment start
laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
```
* To list down and monitor the running containers:
```bash
laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 ps
# With status
docker ps -a
@@ -72,43 +43,20 @@ laconic-so deployment --dir sushiswap-v3-deployment start
docker logs -f <CONTAINER_ID>
```
* Open the GQL playground at http://localhost:3008/graphql
```graphql
# Example query
{
_meta {
block {
number
timestamp
}
hasIndexingErrors
}
factories {
id
poolCount
}
}
```
## Clean up
Stop all the sushiswap-v3 services running in background:
Stop all the services running in background:
```bash
# Only stop the docker containers
laconic-so deployment --dir sushiswap-v3-deployment stop
# Run 'start' to restart the deployment
laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 down
```
To stop all the sushiswap-v3 services and also delete data:
Clear volumes created by this stack:
```bash
# Stop the docker containers
laconic-so deployment --dir sushiswap-v3-deployment stop --delete-volumes
# List all relevant volumes
docker volume ls -q --filter "name=sushiswap_v3"
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r sushiswap-v3-deployment
# Remove all the listed volumes
docker volume rm $(docker volume ls -q --filter "name=sushiswap_v3")
```
@@ -2,7 +2,7 @@ version: "1.0"
name: sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.7
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.6
containers:
- cerc/watcher-sushiswap-v3
pods:
@@ -33,29 +33,23 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec
### Ports
Edit `uniswap-urbit-app-spec.yml` such that it looks like:
Edit `network` in spec file to map container ports to same ports in host:
```yml
stack: uniswap-urbit-app
deploy-to: compose
...
network:
ports:
proxy-server:
- '4000:4000'
urbit-fake-ship:
- '8080:80'
proxy-server:
- '4000:4000'
ipfs:
- '4001'
- '8081:8080'
- 0.0.0.0:5001:5001
volumes:
urbit_app_builds: ./data/urbit_app_builds
urbit_data: ./data/urbit_data
ipfs-import: ./data/ipfs-import
ipfs-data: ./data/ipfs-data
- '8081:8080'
- '5001:5001'
...
```
Note: Skip the `ipfs` ports if using an externally running IPFS node, set via `config.env`, below.
Note: Skip the `ipfs` ports if need to use an externally running IPFS node
### Data volumes
@@ -327,14 +327,12 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
default_spec_file_content = call_stack_deploy_init(deploy_command_context)
spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type}
if deployer_type == "k8s":
if kube_config:
spec_file_content.update({constants.kube_config_key: kube_config})
else:
if kube_config is None:
error_exit("--kube-config must be supplied with --deploy-to k8s")
if image_registry:
spec_file_content.update({constants.image_registry_key: image_registry})
else:
print("WARNING: --image-registry not specified, only default container registries (eg, Docker Hub) will be available")
if image_registry is None:
error_exit("--image-registry must be supplied with --deploy-to k8s")
spec_file_content.update({constants.kube_config_key: kube_config})
spec_file_content.update({constants.image_registry_key: image_registry})
else:
# Check for --kube-config supplied for non-relevant deployer types
if kube_config is not None:
-23
View File
@@ -29,29 +29,6 @@ def _image_needs_pushed(image: str):
return image.endswith(":local")
def remote_image_exists(remote_repo_url: str, local_tag: str):
docker = DockerClient()
try:
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
result = docker.manifest.inspect(remote_tag)
return True if result else False
except Exception: # noqa: E722
return False
def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
if not additional_tags:
return
if not remote_image_exists(remote_repo_url, local_tag):
raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
docker = DockerClient()
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
def remote_tag_for_image(image: str, remote_repo_url: str):
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
major_parts = image.split("/", 2)
+7 -11
View File
@@ -101,7 +101,7 @@ class ClusterInfo:
)
return service
def get_ingress(self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"):
def get_ingress(self, use_tls=False):
# No ingress for a deployment that has no http-proxy defined, for now
http_proxy_info_list = self.spec.get_http_proxy()
ingress = None
@@ -114,8 +114,8 @@ class ClusterInfo:
host_name = http_proxy_info["host-name"]
rules = []
tls = [client.V1IngressTLS(
hosts=certificate["spec"]["dnsNames"] if certificate else [host_name],
secret_name=certificate["spec"]["secretName"] if certificate else f"{self.app_name}-tls"
hosts=[host_name],
secret_name=f"{self.app_name}-tls"
)] if use_tls else None
paths = []
for route in http_proxy_info["routes"]:
@@ -147,17 +147,13 @@ class ClusterInfo:
tls=tls,
rules=rules
)
ingress_annotations = {
"kubernetes.io/ingress.class": "nginx",
}
if not certificate:
ingress_annotations["cert-manager.io/cluster-issuer"] = cluster_issuer
ingress = client.V1Ingress(
metadata=client.V1ObjectMeta(
name=f"{self.app_name}-ingress",
annotations=ingress_annotations
annotations={
"kubernetes.io/ingress.class": "nginx",
"cert-manager.io/cluster-issuer": "letsencrypt-prod"
}
),
spec=spec
)
+3 -45
View File
@@ -169,39 +169,6 @@ class K8sDeployer(Deployer):
print("Service created:")
print(f"{service_resp}")
def _find_certificate_for_host_name(self, host_name):
all_certificates = self.custom_obj_api.list_namespaced_custom_object(
group="cert-manager.io",
version="v1",
namespace=self.k8s_namespace,
plural="certificates"
)
host_parts = host_name.split(".", 1)
host_as_wild = None
if len(host_parts) == 2:
host_as_wild = f"*.{host_parts[1]}"
now = datetime.utcnow().replace(tzinfo=timezone.utc)
fmt = "%Y-%m-%dT%H:%M:%S%z"
# Walk over all the configured certificates.
for cert in all_certificates["items"]:
dns = cert["spec"]["dnsNames"]
# Check for an exact hostname match or a wildcard match.
if host_name in dns or host_as_wild in dns:
status = cert.get("status", {})
# Check the certificate date.
if "notAfter" in status and "notBefore" in status:
before = datetime.strptime(status["notBefore"], fmt)
after = datetime.strptime(status["notAfter"], fmt)
if before < now < after:
# Check the status is Ready
for condition in status.get("conditions", []):
if "True" == condition.get("status") and "Ready" == condition.get("type"):
return cert
return None
def up(self, detach, services):
if not opts.o.dry_run:
if self.is_kind():
@@ -222,15 +189,8 @@ class K8sDeployer(Deployer):
self._create_volume_data()
self._create_deployment()
http_proxy_info = self.cluster_info.spec.get_http_proxy()
# Note: at present we don't support tls for kind (and enabling tls causes errors)
use_tls = http_proxy_info and not self.is_kind()
certificate = self._find_certificate_for_host_name(http_proxy_info[0]["host-name"]) if use_tls else None
if opts.o.debug:
if certificate:
print(f"Using existing certificate: {certificate}")
ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=use_tls, certificate=certificate)
ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
if ingress:
if opts.o.debug:
print(f"Sending this ingress: {ingress}")
@@ -390,11 +350,9 @@ class K8sDeployer(Deployer):
name=ingress.spec.tls[0].secret_name
)
hostname = ingress.spec.rules[0].host
hostname = ingress.spec.tls[0].hosts[0]
ip = ingress.status.load_balancer.ingress[0].ip
tls = "notBefore: %s; notAfter: %s; names: %s" % (
cert["status"]["notBefore"], cert["status"]["notAfter"], ingress.spec.tls[0].hosts
)
tls = "notBefore: %s, notAfter: %s" % (cert["status"]["notBefore"], cert["status"]["notAfter"])
except: # noqa: E722
pass
+24 -22
View File
@@ -27,9 +27,7 @@ class ResourceLimits:
memory: int = None
storage: int = None
def __init__(self, obj=None):
if obj is None:
obj = {}
def __init__(self, obj={}):
if "cpus" in obj:
self.cpus = float(obj["cpus"])
if "memory" in obj:
@@ -52,9 +50,7 @@ class Resources:
limits: ResourceLimits = None
reservations: ResourceLimits = None
def __init__(self, obj=None):
if obj is None:
obj = {}
def __init__(self, obj={}):
if "reservations" in obj:
self.reservations = ResourceLimits(obj["reservations"])
if "limits" in obj:
@@ -76,9 +72,7 @@ class Spec:
obj: typing.Any
file_path: Path
def __init__(self, file_path: Path = None, obj=None) -> None:
if obj is None:
obj = {}
def __init__(self, file_path: Path = None, obj={}) -> None:
self.file_path = file_path
self.obj = obj
@@ -97,41 +91,49 @@ class Spec:
self.file_path = file_path
def get_image_registry(self):
return self.obj.get(constants.image_registry_key)
return (self.obj[constants.image_registry_key]
if self.obj and constants.image_registry_key in self.obj
else None)
def get_volumes(self):
return self.obj.get(constants.volumes_key, {})
return (self.obj["volumes"]
if self.obj and "volumes" in self.obj
else {})
def get_configmaps(self):
return self.obj.get(constants.configmaps_key, {})
return (self.obj["configmaps"]
if self.obj and "configmaps" in self.obj
else {})
def get_container_resources(self):
return Resources(self.obj.get(constants.resources_key, {}).get("containers", {}))
return Resources(self.obj.get("resources", {}).get("containers", {}))
def get_volume_resources(self):
return Resources(self.obj.get(constants.resources_key, {}).get(constants.volumes_key, {}))
return Resources(self.obj.get("resources", {}).get("volumes", {}))
def get_http_proxy(self):
return self.obj.get(constants.network_key, {}).get(constants.http_proxy_key, [])
return (self.obj[constants.network_key][constants.http_proxy_key]
if self.obj and constants.network_key in self.obj
and constants.http_proxy_key in self.obj[constants.network_key]
else None)
def get_annotations(self):
return self.obj.get(constants.annotations_key, {})
return self.obj.get("annotations", {})
def get_labels(self):
return self.obj.get(constants.labels_key, {})
return self.obj.get("labels", {})
def get_privileged(self):
return "true" == str(self.obj.get(constants.security_key, {}).get("privileged", "false")).lower()
return "true" == str(self.obj.get("security", {}).get("privileged", "false")).lower()
def get_capabilities(self):
return self.obj.get(constants.security_key, {}).get("capabilities", [])
return self.obj.get("security", {}).get("capabilities", [])
def get_deployment_type(self):
return self.obj.get(constants.deploy_to_key)
return self.obj[constants.deploy_to_key]
def is_kubernetes_deployment(self):
return self.get_deployment_type() in [constants.k8s_kind_deploy_type,
constants.k8s_deploy_type]
return self.get_deployment_type() in [constants.k8s_kind_deploy_type, constants.k8s_deploy_type]
def is_kind_deployment(self):
return self.get_deployment_type() in [constants.k8s_kind_deploy_type]
@@ -24,9 +24,8 @@ import uuid
import click
from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
from stack_orchestrator.deploy.webapp import deploy_webapp
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedLogger,
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
build_container_image, push_container_image,
file_hash, deploy_to_k8s, publish_deployment,
hostname_for_deployment_request, generate_hostname_for_app,
@@ -43,18 +42,23 @@ def process_app_deployment_request(
deployment_parent_dir,
kube_config,
image_registry,
force_rebuild,
logger
log_parent_dir
):
logger.log("BEGIN - process_app_deployment_request")
run_id = f"{app_deployment_request.id}-{str(time.time()).split('.')[0]}-{str(uuid.uuid4()).split('-')[0]}"
log_file = None
if log_parent_dir:
log_dir = os.path.join(log_parent_dir, app_deployment_request.id)
if not os.path.exists(log_dir):
os.mkdir(log_dir)
log_file_path = os.path.join(log_dir, f"{run_id}.log")
print(f"Directing build logs to: {log_file_path}")
log_file = open(log_file_path, "wt")
# 1. look up application
app = laconic.get_record(app_deployment_request.attributes.application, require=True)
logger.log(f"Retrieved app record {app_deployment_request.attributes.application}")
# 2. determine dns
requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
logger.log(f"Determined requested name: {requested_name}")
# HACK
if "." in requested_name:
@@ -72,7 +76,7 @@ def process_app_deployment_request(
matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True))
if matched_owner:
logger.log(f"Matched DnsRecord ownership: {matched_owner}")
print("Matched DnsRecord ownership:", matched_owner)
else:
raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
(dns_record.id, app_deployment_request.id))
@@ -96,9 +100,7 @@ def process_app_deployment_request(
deployment_record = laconic.get_record(app_deployment_crn)
deployment_dir = os.path.join(deployment_parent_dir, fqdn)
deployment_config_file = os.path.join(deployment_dir, "config.env")
# TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
app_image_shared_tag = f"laconic-webapp/{app.id}:local"
# b. check for deployment directory (create if necessary)
if not os.path.exists(deployment_dir):
if deployment_record:
@@ -113,26 +115,11 @@ def process_app_deployment_request(
needs_k8s_deploy = False
# 6. build container (if needed)
if not deployment_record or deployment_record.attributes.application != app.id:
# TODO: pull from request
extra_build_args = []
build_container_image(app, deployment_container_tag, extra_build_args, log_file)
push_container_image(deployment_dir, log_file)
needs_k8s_deploy = True
# check if the image already exists
shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
if shared_tag_exists and not force_rebuild:
# simply add our unique tag to the existing image and we are done
logger.log(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}")
add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
logger.log("Tag complete")
else:
extra_build_args = [] # TODO: pull from request
logger.log(f"Building container image {deployment_container_tag}")
build_container_image(app, deployment_container_tag, extra_build_args, logger)
logger.log("Build complete")
logger.log(f"Pushing container image {deployment_container_tag}")
push_container_image(deployment_dir, logger)
logger.log("Push complete")
# The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
logger.log(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}")
add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
logger.log("Tag complete")
# 7. update config (if needed)
if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
@@ -140,13 +127,13 @@ def process_app_deployment_request(
# 8. update k8s deployment
if needs_k8s_deploy:
print("Deploying to k8s")
deploy_to_k8s(
deployment_record,
deployment_dir,
logger
log_file
)
logger.log("Publishing deployment to registry.")
publish_deployment(
laconic,
app,
@@ -155,11 +142,8 @@ def process_app_deployment_request(
dns_record,
dns_crn,
deployment_dir,
app_deployment_request,
logger
app_deployment_request
)
logger.log("Publication complete.")
logger.log("END - process_app_deployment_request")
def load_known_requests(filename):
@@ -196,13 +180,12 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
@click.pass_context
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901
request_id, discover, state_file, only_update_state,
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
include_tags, exclude_tags, force_rebuild, log_dir):
include_tags, exclude_tags, log_dir):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
sys.exit(2)
@@ -308,24 +291,10 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
for r in requests_to_execute:
dump_known_requests(state_file, [r], "DEPLOYING")
status = "ERROR"
run_log_file = None
run_reg_client = laconic
try:
run_id = f"{r.id}-{str(time.time()).split('.')[0]}-{str(uuid.uuid4()).split('-')[0]}"
if log_dir:
run_log_dir = os.path.join(log_dir, r.id)
if not os.path.exists(run_log_dir):
os.mkdir(run_log_dir)
run_log_file_path = os.path.join(run_log_dir, f"{run_id}.log")
print(f"Directing deployment logs to: {run_log_file_path}")
run_log_file = open(run_log_file_path, "wt")
run_reg_client = LaconicRegistryClient(laconic_config, log_file=run_log_file)
logger = TimedLogger(run_id, run_log_file)
logger.log("Processing ...")
process_app_deployment_request(
ctx,
run_reg_client,
laconic,
r,
record_namespace_deployments,
record_namespace_dns,
@@ -333,15 +302,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
os.path.abspath(deployment_parent_dir),
kube_config,
image_registry,
force_rebuild,
logger
log_dir
)
status = "DEPLOYED"
except Exception as e:
logger.log("ERROR: " + str(e))
finally:
if logger:
logger.log(f"DONE with status {status}", show_step_time=False, show_total_time=True)
dump_known_requests(state_file, [r], status)
if run_log_file:
run_log_file.close()
+26 -71
View File
@@ -13,7 +13,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import datetime
import hashlib
import json
import os
@@ -40,38 +39,13 @@ class AttrDict(dict):
return v
class TimedLogger:
def __init__(self, id="", file=None):
self.start = datetime.datetime.now()
self.last = self.start
self.id = id
self.file = file
def log(self, msg, show_step_time=True, show_total_time=False):
prefix = f"{datetime.datetime.utcnow()} - {self.id}"
if show_step_time:
prefix += f" - {datetime.datetime.now() - self.last} (step)"
if show_total_time:
prefix += f" - {datetime.datetime.now() - self.start} (total)"
print(f"{prefix}: {msg}", file=self.file)
if self.file:
self.file.flush()
self.last = datetime.datetime.now()
def logged_cmd(log_file, *vargs):
result = None
def cmd(*vargs):
try:
if log_file:
print(" ".join(vargs), file=log_file)
result = subprocess.run(vargs, capture_output=True)
result.check_returncode()
return result.stdout.decode()
except Exception as err:
if result:
print(result.stderr.decode(), file=log_file)
else:
print(str(err), file=log_file)
print(result.stderr.decode())
raise err
@@ -84,9 +58,8 @@ def match_owner(recordA, *records):
class LaconicRegistryClient:
def __init__(self, config_file, log_file=None):
def __init__(self, config_file):
self.config_file = config_file
self.log_file = log_file
self.cache = AttrDict(
{
"name_or_id": {},
@@ -104,7 +77,7 @@ class LaconicRegistryClient:
args.append("--%s" % k)
args.append(str(v))
results = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args))]
results = [AttrDict(r) for r in json.loads(cmd(*args))]
# Most recent records first
results.sort(key=lambda r: r.createTime)
@@ -142,7 +115,7 @@ class LaconicRegistryClient:
args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name]
parsed = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args))]
parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
if parsed:
self._add_to_cache(parsed)
return parsed[0]
@@ -172,7 +145,7 @@ class LaconicRegistryClient:
name_or_id,
]
parsed = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args))]
parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
if len(parsed):
self._add_to_cache(parsed)
return parsed[0]
@@ -200,22 +173,22 @@ class LaconicRegistryClient:
record_file = open(record_fname, 'w')
yaml.dump(record, record_file)
record_file.close()
print(open(record_fname, 'r').read(), file=self.log_file)
print(open(record_fname, 'r').read())
new_record_id = json.loads(
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
)["id"]
for name in names:
self.set_name(name, new_record_id)
return new_record_id
finally:
logged_cmd(self.log_file, "rm", "-rf", tmpdir)
cmd("rm", "-rf", tmpdir)
def set_name(self, name, record_id):
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
def delete_name(self, name):
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "cns", "name", "delete", name)
cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name)
def file_hash(filename):
@@ -239,7 +212,7 @@ def determine_base_container(clone_dir, app_type="webapp"):
return base_container
def build_container_image(app_record, tag, extra_build_args=[], logger=None):
def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
tmpdir = tempfile.mkdtemp()
try:
@@ -248,33 +221,23 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
repo = random.choice(app_record.attributes.repository)
clone_dir = os.path.join(tmpdir, record_id)
logger.log(f"Cloning repository {repo} to {clone_dir} ...")
print(f"Cloning repository {repo} to {clone_dir} ...")
if ref:
# TODO: Determing branch or hash, and use depth 1 if we can.
git_env = dict(os.environ.copy())
# Never prompt
git_env["GIT_TERMINAL_PROMPT"] = "0"
try:
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=logger.file, stderr=logger.file)
except Exception as e:
logger.log(f"git clone failed. Is the repository {repo} private?")
raise e
try:
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=logger.file, stderr=logger.file)
except Exception as e:
logger.log(f"git checkout failed. Does ref {ref} exist?")
raise e
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=log_file, stderr=log_file)
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=log_file, stderr=log_file)
else:
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=logger.file, stderr=logger.file)
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=log_file, stderr=log_file)
result.check_returncode()
base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
logger.log("Building webapp ...")
print("Building webapp ...")
build_command = [
sys.argv[0],
"--verbose",
"build-webapp",
sys.argv[0], "build-webapp",
"--source-repo", clone_dir,
"--tag", tag,
"--base-container", base_container
@@ -283,31 +246,28 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
build_command.append("--extra-build-args")
build_command.append(" ".join(extra_build_args))
result = subprocess.run(build_command, stdout=logger.file, stderr=logger.file)
result = subprocess.run(build_command, stdout=log_file, stderr=log_file)
result.check_returncode()
finally:
logged_cmd(logger.file, "rm", "-rf", tmpdir)
cmd("rm", "-rf", tmpdir)
def push_container_image(deployment_dir, logger):
logger.log("Pushing images ...")
def push_container_image(deployment_dir, log_file=None):
print("Pushing image ...")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"],
stdout=logger.file, stderr=logger.file)
stdout=log_file, stderr=log_file)
result.check_returncode()
logger.log("Finished pushing images.")
def deploy_to_k8s(deploy_record, deployment_dir, logger):
def deploy_to_k8s(deploy_record, deployment_dir, log_file=None):
if not deploy_record:
command = "up"
else:
command = "update"
logger.log("Deploying to k8s ...")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
stdout=logger.file, stderr=logger.file)
stdout=log_file, stderr=log_file)
result.check_returncode()
logger.log("Finished deploying to k8s.")
def publish_deployment(laconic: LaconicRegistryClient,
@@ -317,8 +277,7 @@ def publish_deployment(laconic: LaconicRegistryClient,
dns_record,
dns_crn,
deployment_dir,
app_deployment_request=None,
logger=None):
app_deployment_request=None):
if not deploy_record:
deploy_ver = "0.0.1"
else:
@@ -348,8 +307,6 @@ def publish_deployment(laconic: LaconicRegistryClient,
if app_deployment_request:
new_dns_record["record"]["request"] = app_deployment_request.id
if logger:
logger.log("Publishing DnsRecord.")
dns_id = laconic.publish(new_dns_record, [dns_crn])
new_deployment_record = {
@@ -369,8 +326,6 @@ def publish_deployment(laconic: LaconicRegistryClient,
if app_deployment_request:
new_deployment_record["record"]["request"] = app_deployment_request.id
if logger:
logger.log("Publishing ApplicationDeploymentRecord.")
deployment_id = laconic.publish(new_deployment_record, [deployment_crn])
return {"dns": dns_id, "deployment": deployment_id}
+1 -2
View File
@@ -17,7 +17,7 @@ import click
from stack_orchestrator.command_types import CommandOptions
from stack_orchestrator.repos import setup_repositories
from stack_orchestrator.build import build_containers, fetch_containers
from stack_orchestrator.build import build_containers
from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy.webapp import (run_webapp,
@@ -52,7 +52,6 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
cli.add_command(setup_repositories.command, "setup-repositories")
cli.add_command(build_containers.command, "build-containers")
cli.add_command(fetch_containers.command, "fetch-containers")
cli.add_command(build_npms.command, "build-npms")
cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(run_webapp.command, "run-webapp")
+1 -8
View File
@@ -83,7 +83,7 @@ $TEST_TARGET_SO --stack test deploy down --delete-volumes
# Basic test of creating a deployment
test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
$TEST_TARGET_SO --stack test deploy init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED,CERC_TEST_PARAM_3=FAST
$TEST_TARGET_SO --stack test deploy init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
# Check the file now exists
if [ ! -f "$test_deployment_spec" ]; then
echo "deploy init test: spec file not present"
@@ -141,13 +141,6 @@ else
echo "deployment compose config test: FAILED"
exit 1
fi
# Check the config variable CERC_TEST_PARAM_3 was passed correctly
if [[ "$log_output_3" == *"Test-param-3: FAST"* ]]; then
echo "deployment config test: passed"
else
echo "deployment config test: FAILED"
exit 1
fi
# Check that the ConfigMap is mounted and contains the expected content.
log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-43
View File
@@ -1,43 +0,0 @@
#!/usr/bin/env bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Laconic registry CLI tests"
env
cat /etc/hosts
# Bit of a hack, test the most recent package
TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
echo "$(date +"%Y-%m-%d %T"): Cloning laconic-registry-cli repository into: $CERC_REPO_BASE_DIR"
$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories --include git.vdb.to/cerc-io/laconic-registry-cli
echo "$(date +"%Y-%m-%d %T"): Starting stack"
TEST_AUCTION_ENABLED=true BASE_DIR=${CERC_REPO_BASE_DIR} $TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd up
echo "$(date +"%Y-%m-%d %T"): Stack started"
# Verify that the fixturenet is up and running
$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps
# Get the fixturenet account address
laconicd_account_address=$(docker exec laconicd-laconicd-1 laconicd keys list | awk '/- address:/ {print $3}')
# Copy over config
docker exec laconicd-cli-1 cp config.yml laconic-registry-cli/
# Wait for the laconid endpoint to come up
echo "Waiting for the RPC endpoint to come up"
docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
# Run the tests
echo "Running the tests"
docker exec -e TEST_ACCOUNT=$laconicd_account_address laconicd-cli-1 sh -c 'cd laconic-registry-cli && yarn && yarn test'
# Clean up
$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd down --delete-volumes
echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
rm -rf $CERC_REPO_BASE_DIR
echo "$(date +"%Y-%m-%d %T"): Test finished"
-1
View File
@@ -12,7 +12,6 @@ cat /etc/hosts
TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
# Set a new unique repo dir
export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO"
echo "$(date +"%Y-%m-%d %T"): Test version command"
reported_version_string=$( $TEST_TARGET_SO version )
+2 -2
View File
@@ -32,14 +32,14 @@ set +e
CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
sleep 3
wget --tries 20 --retry-connrefused --waitretry=3 -O test.before -m http://localhost:3000
wget -t 7 -O test.before -m http://localhost:3000
docker logs $CONTAINER_ID
docker remove -f $CONTAINER_ID
CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
sleep 3
wget --tries 20 --retry-connrefused --waitretry=3 -O test.after -m http://localhost:3000
wget -t 7 -O test.after -m http://localhost:3000
docker logs $CONTAINER_ID
docker remove -f $CONTAINER_ID