Compare commits

..
Author SHA1 Message Date
zramsay 817f0082e8 hack
Lint Checks / Run linter (pull_request) Successful in 44s
Deploy Test / Run deploy test suite (pull_request) Successful in 3m39s
Webapp Test / Run webapp test suite (pull_request) Successful in 2m55s
Smoke Test / Run basic test suite (pull_request) Successful in 3m6s
K8s Deploy Test / Run deploy test suite on kind/k8s (pull_request) Successful in 10m8s
2024-02-23 17:07:49 -05:00
72 changed files with 340 additions and 33866 deletions
@@ -43,19 +43,3 @@ jobs:
run: ./scripts/build_shiv_package.sh
- name: "Run fixturenet-eth tests"
run: ./tests/fixturenet-eth-plugeth/run-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -47,19 +47,3 @@ jobs:
sleep 5
- name: "Run fixturenet-eth tests"
run: ./tests/fixturenet-eth-plugeth/run-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+1 -16
View File
@@ -45,19 +45,4 @@ jobs:
sleep 5
- name: "Run fixturenet-eth tests"
run: ./tests/fixturenet-eth/run-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+1 -19
View File
@@ -11,7 +11,7 @@ on:
jobs:
test:
name: "Run Laconicd fixturenet and Laconic CLI tests"
name: "Run an Laconicd fixturenet test"
runs-on: ubuntu-latest
steps:
- name: 'Update'
@@ -46,21 +46,3 @@ jobs:
run: ./scripts/build_shiv_package.sh
- name: "Run fixturenet-laconicd tests"
run: ./tests/fixturenet-laconicd/run-test.sh
- name: "Run laconic CLI tests"
run: ./tests/fixturenet-laconicd/run-cli-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-16
View File
@@ -19,19 +19,3 @@ jobs:
python-version: '3.8'
- name : "Run flake8"
uses: py-actions/flake8@v2
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-16
View File
@@ -54,19 +54,3 @@ jobs:
# Hack using endsWith to workaround Gitea sometimes sending "publish-test" vs "refs/heads/publish-test"
draft: ${{ endsWith('publish-test', github.ref ) }}
files: ./laconic-so
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+1 -16
View File
@@ -51,19 +51,4 @@ jobs:
source /opt/bash-utils/cgroup-helper.sh
join_cgroup
./tests/container-registry/run-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+1 -16
View File
@@ -49,19 +49,4 @@ jobs:
source /opt/bash-utils/cgroup-helper.sh
join_cgroup
./tests/database/run-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-16
View File
@@ -47,19 +47,3 @@ jobs:
sleep 5
- name: "Run deploy tests"
run: ./tests/deploy/run-deploy-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+1 -16
View File
@@ -51,19 +51,4 @@ jobs:
source /opt/bash-utils/cgroup-helper.sh
join_cgroup
./tests/k8s-deploy/run-deploy-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-16
View File
@@ -49,19 +49,3 @@ jobs:
sleep 5
- name: "Run webapp tests"
run: ./tests/webapp-test/run-webapp-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+2 -16
View File
@@ -47,19 +47,5 @@ jobs:
sleep 5
- name: "Run smoke tests"
run: ./tests/smoke-test/run-smoke-test.sh
- name: Notify Vulcanize Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
- name: Notify DeepStack Slack on CI failure
if: ${{ always() && github.ref_name == 'main' }}
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -1,6 +1,3 @@
Change this file to trigger running the fixturenet-laconicd-test CI job
Trigger
Trigger
Trigger
Trigger
Trigger
-9
View File
@@ -1,9 +0,0 @@
# Fetching pre-built container images
When Stack Orchestrator deploys a stack containing a suite of one or more containers it expects images for those containers to be on the local machine with a tag of the form `<image-name>:local` Images for these containers can be built from source (and optionally base container images from public registries) with the `build-containers` subcommand.
However, the task of building a large number of containers from source may consume considerable time and machine resources. This is where the `fetch-containers` subcommand steps in. It is designed to work exactly like `build-containers` but instead the images, pre-built, are fetched from an image registry then re-tagged for deployment. It can be used in place of `build-containers` for any stack provided the necessary containers, built for the local machine architecture (e.g. arm64 or x86-64) have already been published in an image registry.
## Usage
To use `fetch-containers`, provide an image registry path, a username and token/password with read access to the registry, and optionally specify `--force-local-overwrite`. If this argument is not specified, if there is already a locally built or previously fetched image for a stack container on the machine, it will not be overwritten and a warning issued.
```
$ laconic-so --stack mobymask-v3-demo fetch-containers --image-registry git.vdb.to/cerc-io --registry-username <registry-user> --registry-token <registry-token> --force-local-overwrite
```
+20 -3
View File
@@ -25,13 +25,13 @@ import sys
from decouple import config
import subprocess
import click
import importlib.resources
from pathlib import Path
from stack_orchestrator.opts import opts
from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit
from stack_orchestrator.base import get_npm_registry_url
from stack_orchestrator.build.build_types import BuildContext
from stack_orchestrator.build.publish import publish_image
from stack_orchestrator.build.build_util import get_containers_in_scope
# TODO: find a place for this
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -149,7 +149,24 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
if not image_registry:
error_exit("--image-registry must be supplied with --publish-images")
containers_in_scope = get_containers_in_scope(stack)
# See: https://stackoverflow.com/a/20885799/1701505
from stack_orchestrator import data
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
all_containers = container_list_file.read().splitlines()
containers_in_scope = []
if stack:
stack_config = get_parsed_stack_config(stack)
if "containers" not in stack_config or stack_config["containers"] is None:
warn_exit(f"stack {stack} does not define any containers")
containers_in_scope = stack_config['containers']
else:
containers_in_scope = all_containers
if opts.o.verbose:
print(f'Containers: {containers_in_scope}')
if stack:
print(f"Stack: {stack}")
container_build_env = make_container_build_env(dev_root_path,
container_build_dir,
-43
View File
@@ -1,43 +0,0 @@
# Copyright © 2024 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import importlib.resources
from stack_orchestrator.opts import opts
from stack_orchestrator.util import get_parsed_stack_config, warn_exit
def get_containers_in_scope(stack: str):
# See: https://stackoverflow.com/a/20885799/1701505
from stack_orchestrator import data
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
all_containers = container_list_file.read().splitlines()
containers_in_scope = []
if stack:
stack_config = get_parsed_stack_config(stack)
if "containers" not in stack_config or stack_config["containers"] is None:
warn_exit(f"stack {stack} does not define any containers")
containers_in_scope = stack_config['containers']
else:
containers_in_scope = all_containers
if opts.o.verbose:
print(f'Containers: {containers_in_scope}')
if stack:
print(f"Stack: {stack}")
return containers_in_scope
+12 -32
View File
@@ -21,13 +21,11 @@
# TODO: display the available list of containers; allow re-build of either all or specific containers
import os
import sys
from decouple import config
import click
from pathlib import Path
from stack_orchestrator.build import build_containers
from stack_orchestrator.deploy.webapp.util import determine_base_container, TimedLogger
from stack_orchestrator.deploy.webapp.util import determine_base_container
from stack_orchestrator.build.build_types import BuildContext
@@ -40,25 +38,26 @@ from stack_orchestrator.build.build_types import BuildContext
@click.pass_context
def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
'''build the specified webapp container'''
logger = TimedLogger()
quiet = ctx.obj.quiet
debug = ctx.obj.debug
verbose = ctx.obj.verbose
dry_run = ctx.obj.dry_run
debug = ctx.obj.debug
local_stack = ctx.obj.local_stack
stack = ctx.obj.stack
continue_on_error = ctx.obj.continue_on_error
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
if local_stack:
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
logger.log(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
else:
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
if verbose:
logger.log(f'Dev Root is: {dev_root_path}')
if not quiet:
print(f'Dev Root is: {dev_root_path}')
if not base_container:
base_container = determine_base_container(source_repo)
@@ -67,9 +66,6 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
force_rebuild, extra_build_args)
if verbose:
logger.log(f"Building base container: {base_container}")
build_context_1 = BuildContext(
stack,
base_container,
@@ -77,13 +73,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
container_build_env,
dev_root_path,
)
ok = build_containers.process_container(build_context_1)
if not ok:
logger.log("ERROR: Build failed.")
sys.exit(1)
if verbose:
logger.log(f"Base container {base_container} build finished.")
build_containers.process_container(build_context_1)
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
@@ -93,12 +83,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
"Dockerfile.webapp")
if not tag:
webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
tag = f"cerc/{webapp_name}:local"
container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
if verbose:
logger.log(f"Building app container: {tag}")
container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
else:
container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
build_context_2 = BuildContext(
stack,
@@ -107,11 +94,4 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
container_build_env,
dev_root_path,
)
ok = build_containers.process_container(build_context_2)
if not ok:
logger.log("ERROR: Build failed.")
sys.exit(1)
if verbose:
logger.log(f"App container {base_container} build finished.")
logger.log("build-webapp complete", show_step_time=False, show_total_time=True)
build_containers.process_container(build_context_2)
@@ -1,195 +0,0 @@
# Copyright © 2024 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import click
from dataclasses import dataclass
import json
import platform
from python_on_whales import DockerClient
from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList
from python_on_whales.utils import run
import requests
from typing import List
from stack_orchestrator.opts import opts
from stack_orchestrator.util import include_exclude_check, error_exit
from stack_orchestrator.build.build_util import get_containers_in_scope
# Experimental fetch-container command
@dataclass
class RegistryInfo:
registry: str
registry_username: str
registry_token: str
# Extending this code to support the --verbose option, cnosider contributing upstream
# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129
class ExtendedManifestCLI(ManifestCLI):
def inspect_verbose(self, x: str) -> ManifestList:
"""Returns a Docker manifest list object."""
json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x])
return json.loads(json_str)
def _local_tag_for(container: str):
return f"{container}:local"
# See: https://docker-docs.uclv.cu/registry/spec/api/
# Emulate this:
# $ curl -u "my-username:my-token" -X GET "https://<container-registry-hostname>/v2/cerc-io/cerc/test-container/tags/list"
# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
# registry looks like: git.vdb.to/cerc-io
registry_parts = registry_info.registry.split("/")
url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
if opts.o.debug:
print(f"Fetching tags from: {url}")
response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token))
if response.status_code == 200:
tag_info = response.json()
if opts.o.debug:
print(f"container tags list: {tag_info}")
tags_array = tag_info["tags"]
return tags_array
else:
error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
def _find_latest(candidate_tags: List[str]):
# Lex sort should give us the latest first
sorted_candidates = sorted(candidate_tags)
if opts.o.debug:
print(f"sorted candidates: {sorted_candidates}")
return sorted_candidates[-1]
def _filter_for_platform(container: str,
registry_info: RegistryInfo,
tag_list: List[str]) -> List[str] :
filtered_tags = []
this_machine = platform.machine()
# Translate between Python and docker platform names
if this_machine == "x86_64":
this_machine = "amd64"
if this_machine == "aarch64":
this_machine = "arm64"
if opts.o.debug:
print(f"Python says the architecture is: {this_machine}")
docker = DockerClient()
for tag in tag_list:
remote_tag = f"{registry_info.registry}/{container}:{tag}"
manifest_cmd = ExtendedManifestCLI(docker.client_config)
manifest = manifest_cmd.inspect_verbose(remote_tag)
if opts.o.debug:
print(f"manifest: {manifest}")
image_architecture = manifest["Descriptor"]["platform"]["architecture"]
if opts.o.debug:
print(f"image_architecture: {image_architecture}")
if this_machine == image_architecture:
filtered_tags.append(tag)
if opts.o.debug:
print(f"Tags filtered for platform: {filtered_tags}")
return filtered_tags
def _get_latest_image(container: str, registry_info: RegistryInfo):
all_tags = _get_tags_for_container(container, registry_info)
tags_for_platform = _filter_for_platform(container, registry_info, all_tags)
if len(tags_for_platform) > 0:
latest_tag = _find_latest(tags_for_platform)
return f"{container}:{latest_tag}"
else:
return None
def _fetch_image(tag: str, registry_info: RegistryInfo):
docker = DockerClient()
remote_tag = f"{registry_info.registry}/{tag}"
if opts.o.debug:
print(f"Attempting to pull this image: {remote_tag}")
docker.image.pull(remote_tag)
def _exists_locally(container: str):
docker = DockerClient()
return docker.image.exists(_local_tag_for(container))
def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
docker = DockerClient()
docker.image.tag(f"{registry}/{remote_tag}", local_tag)
@click.command()
@click.option('--include', help="only fetch these containers")
@click.option('--exclude', help="don\'t fetch these containers")
@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
@click.option("--registry-username", required=True, help="Specify the image registry username")
@click.option("--registry-token", required=True, help="Specify the image registry access token")
@click.pass_context
def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token):
'''EXPERIMENTAL: fetch the images for a stack from remote registry'''
registry_info = RegistryInfo(image_registry, registry_username, registry_token)
docker = DockerClient()
if not opts.o.quiet:
print("Logging into container registry:")
docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
# Generate list of target containers
stack = ctx.obj.stack
containers_in_scope = get_containers_in_scope(stack)
all_containers_found = True
for container in containers_in_scope:
local_tag = _local_tag_for(container)
if include_exclude_check(container, include, exclude):
if opts.o.debug:
print(f"Processing: {container}")
# For each container, attempt to find the latest of a set of
# images with the correct name and platform in the specified registry
image_to_fetch = _get_latest_image(container, registry_info)
if not image_to_fetch:
print(f"Warning: no image found to fetch for container: {container}")
all_containers_found = False
continue
if opts.o.debug:
print(f"Fetching: {image_to_fetch}")
_fetch_image(image_to_fetch, registry_info)
# Now check if the target container already exists exists locally already
if (_exists_locally(container)):
if not opts.o.quiet:
print(f"Container image {container} already exists locally")
# if so, fail unless the user specified force-local-overwrite
if (force_local_overwrite):
# In that case remove the existing :local tag
if not opts.o.quiet:
print(f"Warning: overwriting local tag from this image: {container} because "
"--force-local-overwrite was specified")
else:
if not opts.o.quiet:
print(f"Skipping local tagging for this image: {container} because that would "
"overwrite an existing :local tagged image, use --force-local-overwrite to do so.")
continue
# Tag the fetched image with the :local tag
_add_local_tag(image_to_fetch, image_registry, local_tag)
else:
if opts.o.verbose:
print(f"Excluding: {container}")
if not all_containers_found:
print("Warning: couldn't find usable images for one or more containers, this stack will not deploy")
@@ -1,80 +0,0 @@
# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
services:
# generate jwt.txt if it's absent
generate-jwt:
image: blastio/openssl
volumes:
- blast-data:/blast:rw
command: >
sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
# initialise geth db
geth-init:
image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
volumes:
- blast-data:/blast:rw
- ../config/fixturenet-blast/genesis.json:/blast/genesis.json
entrypoint: /bin/sh
command: >
-c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
depends_on:
generate-jwt:
condition: service_completed_successfully
env_file:
- ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
blast-geth:
image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
volumes:
- blast-data:/blast
ports:
- "9545"
- "9546"
command: >
--datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
--http
--http.corsdomain="*"
--http.vhosts="*"
--http.addr=0.0.0.0
--http.port=9545
--http.api=web3,debug,eth,txpool,net,engine
--ws
--ws.addr=0.0.0.0
--ws.port=9546
--ws.origins="*"
--ws.api=debug,eth,txpool,net,engine
--authrpc.addr="0.0.0.0"
--authrpc.port="8551"
--authrpc.vhosts="*"
--authrpc.jwtsecret=/blast/jwt.txt
--syncmode=full
--gcmode=archive
--nodiscover
--maxpeers=0
--rollup.disabletxpoolgossip=true
env_file:
- ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
depends_on:
geth-init:
condition: service_completed_successfully
op-node:
image: blastio/blast-optimism:${NETWORK:-testnet-sepolia}
volumes:
- blast-data:/blast
- ../config/fixturenet-blast/rollup.json:/blast/rollup.json
ports:
- "9003"
command: >
op-node
--l1="${CERC_L1_RPC}"
--l1.rpckind="any"
--l1.trustrpc=true
--l2="http://blast-geth:8551"
--l2.jwt-secret=/blast/jwt.txt
--rollup.config="/blast/rollup.json"
depends_on:
- blast-geth
env_file:
- ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
volumes:
blast-data:
@@ -3,9 +3,6 @@ services:
restart: unless-stopped
image: cerc/laconicd:local
command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
environment:
TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED}
TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY}
volumes:
# The cosmos-sdk node's database directory:
- laconicd-data:/root/.laconicd
@@ -28,7 +25,6 @@ services:
image: cerc/laconic-registry-cli:local
volumes:
- ../config/fixturenet-laconicd/registry-cli-config-template.yml:/registry-cli-config-template.yml
- ${BASE_DIR:-~/cerc}/laconic-registry-cli:/laconic-registry-cli
volumes:
laconicd-data:
@@ -1,86 +0,0 @@
# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
services:
# generate jwt.txt if it's absent
generate-jwt:
image: blastio/openssl
volumes:
- blast-data:/blast:rw
command: >
sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
# initialise geth db
geth-init:
image: blastio/blast-geth:${NETWORK:-mainnet}
volumes:
- blast-data:/blast:rw
entrypoint: /bin/sh
command: >
-c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
depends_on:
generate-jwt:
condition: service_completed_successfully
env_file:
- ../config/mainnet-blast/${NETWORK:-mainnet}.config
blast-geth:
image: blastio/blast-geth:${NETWORK:-mainnet}
volumes:
- blast-data:/blast
ports:
- "9545"
- "9546"
- "6060"
command: >
--datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
--http
--http.corsdomain="*"
--http.vhosts="*"
--http.addr=0.0.0.0
--http.port=9545
--http.api=web3,debug,eth,txpool,net,engine
--ws
--ws.addr=0.0.0.0
--ws.port=9546
--ws.origins="*"
--ws.api=debug,eth,txpool,net,engine
--authrpc.addr="0.0.0.0"
--authrpc.port="8551"
--authrpc.vhosts="*"
--authrpc.jwtsecret=/blast/jwt.txt
--syncmode=full
--metrics
--metrics.addr=0.0.0.0
--gcmode=archive
--nodiscover
--maxpeers=0
--rollup.disabletxpoolgossip=true
--override.canyon="${OVERRIDE_CANYON:-0}"
--override.ecotone="${OVERRIDE_ECOTONE:-1716843599}"
env_file:
- ../config/mainnet-blast/${NETWORK:-mainnet}.config
depends_on:
geth-init:
condition: service_completed_successfully
op-node:
image: blastio/blast-optimism:${NETWORK:-mainnet}
volumes:
- blast-data:/blast
ports:
- "9003"
- "7300"
command: >
op-node
--l1="${L1_RPC:-https://eth-mainnet-1.vdb.to/}"
--l1.beacon="${L1_RPC_BEACON:-https://eth-mainnet-1.vdb.to/}"
--l1.rpckind="any"
--l1.trustrpc=true
--l2="${L2_RPC:-http://blast-geth:8551}"
--l2.jwt-secret=/blast/jwt.txt
--rollup.config="/blast/rollup.json"
--metrics.enabled
depends_on:
- blast-geth
env_file:
- ../config/mainnet-blast/${NETWORK:-mainnet}.config
volumes:
blast-data:
@@ -1,76 +0,0 @@
version: '3.2'
services:
ajna-watcher-db:
restart: unless-stopped
image: postgres:14-alpine
environment:
- POSTGRES_USER=vdbm
- POSTGRES_MULTIPLE_DATABASES=ajna-watcher,ajna-watcher-job-queue
- POSTGRES_EXTENSION=ajna-watcher-job-queue:pgcrypto
- POSTGRES_PASSWORD=password
volumes:
- ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
- ajna_watcher_db_data:/var/lib/postgresql/data
ports:
- "5432"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "5432"]
interval: 20s
timeout: 5s
retries: 15
start_period: 10s
ajna-watcher-job-runner:
restart: unless-stopped
depends_on:
ajna-watcher-db:
condition: service_healthy
image: cerc/watcher-ajna:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-job-runner.sh"]
volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
- ../config/watcher-ajna/start-job-runner.sh:/app/start-job-runner.sh
ports:
- "9000"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "9000"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
ajna-watcher-server:
restart: unless-stopped
depends_on:
ajna-watcher-db:
condition: service_healthy
ajna-watcher-job-runner:
condition: service_healthy
image: cerc/watcher-ajna:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-server.sh"]
volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
- ../config/watcher-ajna/start-server.sh:/app/start-server.sh
ports:
- "3008"
- "9001"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "3008"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
ajna_watcher_db_data:
@@ -1,2 +0,0 @@
GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.s2.testblast.io
OP_NODE_P2P_BOOTNODES=enr:-J-4QM3GLUFfKMSJQuP1UvuKQe8DyovE7Eaiit0l6By4zjTodkR4V8NWXJxNmlg8t8rP-Q-wp3jVmeAOml8cjMj__ROGAYznzb_HgmlkgnY0gmlwhA-cZ_eHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAiuDqvB-AsVSRmnnWr6OHfjgY8YfNclFy9p02flKzXnOg3RjcIJ2YYN1ZHCCdmE,enr:-J-4QDCVpByqQ8nFqCS9aHicqwUfXgzFDslvpEyYz19lvkHLIdtcIGp2d4q5dxHdjRNTO6HXCsnIKxUeuZSPcEbyVQCGAYznzz0RgmlkgnY0gmlwhANiQfuHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAy3AtF2Jh_aPdOohg506Hjmtx-fQ1AKmu71C7PfkWAw9g3RjcIJ2YYN1ZHCCdmE
@@ -1,57 +0,0 @@
{
"config": {
"chainId": 608943043,
"homesteadBlock": 0,
"eip150Block": 0,
"eip155Block": 0,
"eip158Block": 0,
"byzantiumBlock": 0,
"constantinopleBlock": 0,
"petersburgBlock": 0,
"istanbulBlock": 0,
"muirGlacierBlock": 0,
"berlinBlock": 0,
"londonBlock": 0,
"arrowGlacierBlock": 0,
"grayGlacierBlock": 0,
"mergeNetsplitBlock": 0,
"shanghaiTime": 0,
"bedrockBlock": 0,
"regolithTime": 0,
"canyonTime": 0,
"terminalTotalDifficulty": 0,
"terminalTotalDifficultyPassed": true,
"optimism": {
"eip1559Elasticity": 6,
"eip1559Denominator": 50,
"eip1559DenominatorCanyon": 250
}
},
"alloc": {
"0000000000000000000000000000000000000000": {
"balance": "0x1"
},
"4200000000000000000000000000000000000000": {
"code": "0x60806040526004361061004e5760003560e01c80633659cfe6146100655780634f1ef286146100855780635c60da1b146100ae5780638f283970146100db578063f851a440146100fb5761005d565b3661005d5761005b610110565b005b61005b610110565b34801561007157600080fd5b5061005b610080366004610521565b6101c8565b61009861009336600461053c565b61020e565b6040516100a591906105bf565b60405180910390f35b3480156100ba57600080fd5b506100c361033e565b6040516001600160a01b0390911681526020016100a5565b3480156100e757600080fd5b5061005b6100f6366004610521565b6103a9565b34801561010757600080fd5b506100c36103e4565b600061013a7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b90506001600160a01b0381166101a55760405162461bcd60e51b815260206004820152602560248201527f50726f78793a20696d706c656d656e746174696f6e206e6f7420696e697469616044820152641b1a5e995960da1b60648201526084015b60405180910390fd5b3660008037600080366000845af43d6000803e806101c2573d6000fd5b503d6000f35b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806101f5575033155b156102065761020381610432565b50565b610203610110565b60606102266000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610243575033155b1561032f5761025184610432565b600080856001600160a01b0316858560405161026e929190610614565b600060405180830381855af49150503d80600081146102a9576040519150601f19603f3d011682016040523d82523d6000602084013e6102ae565b606091505b5091509150816103265760405162461bcd60e51b815260206004820152603960248201527f50726f78793a2064656c656761746563616c6c20746f206e657720696d706c6560448201527f6d656e746174696f6e20636f6e7472616374206661696c656400000000000000606482015260840161019c565b91506103379050565b610337610110565b9392505050565b60006103566000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610373575033155b1561039e57507f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b6103a6610110565b90565b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806103d6575033155b15610206576102038161048e565b60006103fc6000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610419575033155b1561039e57506000805160206106258339815191525490565b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8181556040516001600160a01b038316907fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b90600090a25050565b60006104a66000805160206106258339815191525490565b600080516020610625833981519152838155604080516001600160a01b0380851682528616602082015292935090917f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f910160405180910390a1505050565b80356001600160a01b038116811461051c57600080fd5b919050565b60006020828403121561053357600080fd5b61033782610505565b60008060006040848603121561055157600080fd5b61055a84610505565b9250602084013567ffffffffffffffff8082111561057757600080fd5b818601915086601f83011261058b57600080fd5b81358181111561059a57600080fd5b8760208285010111156105ac57600080fd5b6020830194508093505050509250925092565b600060208083528351808285015260005b818110156105ec578581018301518582016040015282016105d0565b818111156105fe576000604083870101525b50601f01601f1916929092016040019392505050565b818382376000910190815291905056feb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103a164736f6c634300080f000a",
"storage": {
"0x360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc": "0x000000000000000000000000c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d30000",
"0xb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103": "0x0000000000000000000000004200000000000000000000000000000000000018"
},
"balance": "0x0",
"flags": 1
}
},
"nonce": "0x0",
"timestamp": "0x659b7460",
"extraData": "0x424544524f434b",
"gasLimit": "0x1c9c380",
"difficulty": "0x0",
"mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"coinbase": "0x4200000000000000000000000000000000000011",
"number": "0x0",
"gasUsed": "0x0",
"parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"baseFeePerGas": "0x3b9aca00",
"excessBlobGas": null,
"blobGasUsed": null
}
@@ -1,31 +0,0 @@
{
"genesis": {
"l1": {
"hash": "0x17728cf4d8e0b4f292d2390a869fd7c632d39e72efb00ca3462b4387c6aa2437",
"number": 5044255
},
"l2": {
"hash": "0x26a1c0faad7b041f34569a1bb383f00ab74b335883a44bed53e9f41ced5fd906",
"number": 0
},
"l2_time": 1704686688,
"system_config": {
"batcherAddr": "0xba26fee2fa917443e05e65de8d4350bcd2f59222",
"overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
"scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
"gasLimit": 30000000
}
},
"block_time": 2,
"max_sequencer_drift": 600,
"seq_window_size": 3600,
"channel_timeout": 300,
"l1_chain_id": 11155111,
"l2_chain_id": 608943043,
"regolith_time": 0,
"canyon_time": 0,
"batch_inbox_address": "0x1c3b85a2108784eab6a4bf56cdd6f722e415b331",
"deposit_contract_address": "0x2757e4430e694f27b73ec9c02257cab3a498c8c5",
"l1_system_config_address": "0x329faf078c364a316e08bf6a17b7eee6ae75a613",
"protocol_versions_address": "0x0000000000000000000000000000000000000000"
}
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large Load Diff
@@ -1,32 +0,0 @@
POSTGRES_DB=keycloak
POSTGRES_USER=keycloak
POSTGRES_PASSWORD=keycloak
# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml
PGPORT=35432
KC_DB=postgres
KC_DB_URL_HOST=keycloak-db
KC_DB_URL_PORT=${PGPORT}
KC_DB_URL_DATABASE=${POSTGRES_DB}
KC_DB_USERNAME=${POSTGRES_USER}
KC_DB_PASSWORD=${POSTGRES_PASSWORD}
KC_DB_SCHEMA=public
KC_HOSTNAME=localhost
KC_HTTP_ENABLED="true"
KC_HTTP_RELATIVE_PATH="/auth"
KC_HOSTNAME_STRICT_HTTPS="false"
KEYCLOAK_ADMIN=admin
KEYCLOAK_ADMIN_PASSWORD=admin
X_API_CHECK_REALM=cerc
X_API_CHECK_CLIENT_ID="%user_id%"
# keycloak-reg-api
CERC_KCUSERREG_LISTEN_PORT=9292
CERC_KCUSERREG_LISTEN_ADDR='0.0.0.0'
CERC_KCUSERREG_API_URL='http://keycloak:8080/auth'
CERC_KCUSERREG_REG_USER="${KEYCLOAK_ADMIN}"
CERC_KCUSERREG_REG_PW="${KEYCLOAK_ADMIN_PASSWORD}"
CERC_KCUSERREG_REG_CLIENT_ID='admin-cli'
CERC_KCUSERREG_TARGET_REALM=cerc
CERC_KCUSERREG_TARGET_GROUPS=eth
CERC_KCUSERREG_CREATE_ENABLED=true
@@ -1,33 +0,0 @@
# Enable startup script debug output.
CERC_SCRIPT_DEBUG=false
# Specify any other lighthouse CLI options.
LIGHTHOUSE_OPTS=""
# Override the advertised public IP (optional)
# --enr-address
#LIGHTHOUSE_ENR_ADDRESS=""
# --checkpoint-sync-url
LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc"
# --checkpoint-sync-url-timeout
LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300
# --datadir
LIGHTHOUSE_DATADIR=/data
# --debug-level
LIGHTHOUSE_DEBUG_LEVEL=info
# --http-port
LIGHTHOUSE_HTTP_PORT=5052
# --execution-jwt
LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
# --metrics-port
LIGHTHOUSE_METRICS_PORT=5054
# --port --enr-udp-port --enr-tcp-port
LIGHTHOUSE_NETWORK_PORT=9000
@@ -1,2 +0,0 @@
GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.blast.io
OP_NODE_P2P_BOOTNODES=enr:-J64QGwHl9uYLfC_cnmxSA6wQH811nkOWJDWjzxqkEUlJoZHWvI66u-BXgVcPCeMUmg0dBpFQAPotFchG67FHJMZ9OSGAY3d6wevgmlkgnY0gmlwhANizeSHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaECg4pk0cskPAyJ7pOmo9E6RqGBwV-Lex4VS9a3MQvu7PWDdGNwgnZhg3VkcIJ2YQ,enr:-J64QDge2jYBQtcNEpRqmKfci5E5BHAhNBjgv4WSdwH1_wPqbueq2bDj38-TSW8asjy5lJj1Xftui6Or8lnaYFCqCI-GAY3d6wf3gmlkgnY0gmlwhCO2D9yHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaEDo4aCTq7pCEN8om9U5n_VyWdambGnQhwHNwKc8o-OicaDdGNwgnZhg3VkcIJ2YQ
@@ -1,32 +0,0 @@
{
"genesis": {
"l1": {
"hash": "0xfcfb8d586bdae763f1189988789211c69eb893a895e7ba48be3ca6289f0941b7",
"number": 19300102
},
"l2": {
"hash": "0xb689b35ef29d0bec5816938e0e52683c7257d2e325420ea69b739a2be4754b89",
"number": 0
},
"l2_time": 1708809815,
"system_config": {
"batcherAddr": "0x415c8893d514f9bc5211d36eeda4183226b84aa7",
"overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
"scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
"gasLimit": 30000000
}
},
"block_time": 2,
"max_sequencer_drift": 600,
"seq_window_size": 3600,
"channel_timeout": 300,
"l1_chain_id": 1,
"l2_chain_id": 81457,
"regolith_time": 0,
"canyon_time": 0,
"batch_inbox_address": "0xff00000000000000000000000000000000081457",
"deposit_contract_address": "0x0ec68c5b10f21effb74f2a5c61dfe6b08c0db6cb",
"l1_system_config_address": "0x5531dcff39ec1ec727c4c5d2fc49835368f805a9",
"protocol_versions_address": "0x0000000000000000000000000000000000000000"
}
@@ -1,30 +0,0 @@
#!/bin/bash
if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
set -x
fi
ENR_OPTS=""
if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
fi
exec lighthouse bn \
--checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
--checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \
--datadir "$LIGHTHOUSE_DATADIR" \
--debug-level $LIGHTHOUSE_DEBUG_LEVEL \
--disable-deposit-contract-sync \
--disable-upnp \
--enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
--enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
--execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \
--execution-jwt /etc/mainnet-eth/jwtsecret \
--http \
--http-address 0.0.0.0 \
--http-port $LIGHTHOUSE_HTTP_PORT \
--metrics \
--metrics-address=0.0.0.0 \
--metrics-port $LIGHTHOUSE_METRICS_PORT \
--network mainnet \
--port $LIGHTHOUSE_NETWORK_PORT \
$ENR_OPTS $LIGHTHOUSE_OPTS
@@ -50,6 +50,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -126,6 +142,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -202,6 +234,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -278,6 +326,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -354,6 +418,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -430,6 +510,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -506,6 +602,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -582,6 +694,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -660,6 +788,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -736,6 +880,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -1,20 +0,0 @@
#!/bin/sh
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
set -u
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
echo "Running job-runner..."
DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js
@@ -1,20 +0,0 @@
#!/bin/sh
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
set -u
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
echo "Running server..."
DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js
@@ -1,98 +0,0 @@
[server]
host = "0.0.0.0"
port = 3008
kind = "active"
gqlPath = "/"
# Checkpointing state.
checkpointing = true
# Checkpoint interval in number of blocks.
checkpointInterval = 2000
# Enable state creation
# CAUTION: Disable only if state creation is not desired or can be filled subsequently
enableState = false
subgraphPath = "./subgraph-build"
# Interval to restart wasm instance periodically
wasmRestartBlocksInterval = 20
# Interval in number of blocks at which to clear entities cache.
clearEntitiesCacheInterval = 1000
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Flag to specify whether RPC endpoint supports block hash as block tag parameter
rpcSupportsBlockHashParam = false
# GQL cache settings
[server.gqlCache]
enabled = true
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
[metrics]
host = "0.0.0.0"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"
host = "ajna-watcher-db"
port = 5432
database = "ajna-watcher"
username = "vdbm"
password = "password"
synchronize = true
logging = false
[upstream]
[upstream.ethServer]
rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
# Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
rpcClient = true
# Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
isFEVM = true
# Boolean flag to filter event logs by contracts
filterLogsByAddresses = true
# Boolean flag to filter event logs by topics
filterLogsByTopics = true
[upstream.cache]
name = "requests"
enabled = false
deleteOnStart = false
[jobQueue]
dbConnectionString = "postgres://vdbm:password@ajna-watcher-db/ajna-watcher-job-queue"
maxCompletionLagInSecs = 300
jobDelayInMilliSecs = 100
eventsInBatch = 50
subgraphEventsOrder = true
# Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
blockDelayInMilliSecs = 30000
# Boolean to switch between modes of processing events when starting the server.
# Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
# Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
useBlockRanges = true
# Block range in which logs are fetched during historical blocks processing
historicalLogsBlockRange = 2000
# Max block range of historical processing after which it waits for completion of events processing
# If set to -1 historical processing does not wait for events processing and completes till latest canonical block
historicalMaxFetchAhead = 10000
@@ -26,8 +26,6 @@ RUN \
&& su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
# Install semver
&& su ${USERNAME} -c "umask 0002 && npm install -g semver" \
# Install pnpm
&& su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
&& npm cache clean --force > /dev/null 2>&1
# [Optional] Uncomment this section to install additional OS packages.
@@ -37,9 +35,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# [Optional] Uncomment if you want to install more global node modules
# RUN su node -c "npm install -g <your-package-list-here>"
# We do this to get a yq binary from the published container, for the correct architecture we're building here
COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
# Expose port for http
EXPOSE 80
@@ -11,14 +11,8 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}
docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
rc=$?
if [ $rc -ne 0 ]; then
echo "BUILD FAILED" 1>&2
exit $rc
fi
if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
cat <<EOF
#################################################################
@@ -10,12 +10,10 @@ TRG_DIR="${3:-.next-r}"
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
if [ -f "pnpm-lock.yaml" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
else
if [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=npm
else
CERC_BUILD_TOOL=yarn
fi
fi
@@ -9,15 +9,17 @@ CERC_MIN_NEXTVER=13.4.2
CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
if [ -f "pnpm-lock.yaml" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
if [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
else
CERC_BUILD_TOOL=npm
fi
fi
echo "for building the mars v2 frontend"
npm install --save dotenv
yarn install-charting-library
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
WORK_DIR="${1:-/app}"
@@ -115,7 +117,7 @@ if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VE
mv package.json.$$ package.json
fi
time $CERC_BUILD_TOOL install || exit 1
$CERC_BUILD_TOOL install || exit 1
CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json`
@@ -138,9 +140,9 @@ to use for the build with:
EOF
cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$
mv package.json.$$ package.json
time $CERC_BUILD_TOOL install || exit 1
$CERC_BUILD_TOOL install || exit 1
fi
time $CERC_BUILD_TOOL run cerc_compile || exit 1
$CERC_BUILD_TOOL run cerc_compile || exit 1
exit 0
@@ -16,9 +16,7 @@ trap ctrl_c INT
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
if [ -f "pnpm-lock.yaml" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
CERC_BUILD_TOOL=yarn
else
CERC_BUILD_TOOL=npm
@@ -1,10 +0,0 @@
FROM node:18.17.1-alpine3.18
RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
WORKDIR /app
COPY . .
RUN echo "Installing dependencies and building ajna-watcher-ts" && \
yarn && yarn build
@@ -1,9 +0,0 @@
#!/usr/bin/env bash
# Build cerc/watcher-ajna
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
# See: https://stackoverflow.com/a/246128/1701505
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/watcher-ajna:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/ajna-watcher-ts
@@ -1,6 +1,6 @@
# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
ARG VARIANT=20-bullseye-slim
ARG VARIANT=20-bullseye
FROM node:${VARIANT}
ARG USERNAME=node
@@ -24,10 +24,6 @@ RUN \
&& su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
# Install eslint
&& su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
# Install semver
&& su ${USERNAME} -c "umask 0002 && npm install -g semver" \
# Install pnpm
&& su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
&& npm cache clean --force > /dev/null 2>&1
# [Optional] Uncomment this section to install additional OS packages.
@@ -1,12 +1,11 @@
FROM cerc/webapp-base:local as builder
ARG CERC_BUILD_TOOL
ARG CERC_BUILD_OUTPUT_DIR
WORKDIR /app
COPY . .
RUN rm -rf node_modules build dist .next*
RUN /scripts/build-app.sh /app /data
RUN rm -rf node_modules build .next*
RUN /scripts/build-app.sh /app build /data
FROM cerc/webapp-base:local
COPY --from=builder /data /data
@@ -11,14 +11,8 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local}
docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
rc=$?
if [ $rc -ne 0 ]; then
echo "BUILD FAILED" 1>&2
exit $rc
fi
if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
cat <<EOF
#################################################################
@@ -7,10 +7,9 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
CERC_BUILD_OUTPUT_DIR="${CERC_BUILD_OUTPUT_DIR}"
WORK_DIR="${1:-/app}"
DEST_DIR="${2:-/data}"
OUTPUT_DIR="${2:-build}"
DEST_DIR="${3:-/data}"
if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
@@ -23,30 +22,18 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
cd "${WORK_DIR}" || exit 1
if [ -z "$CERC_BUILD_TOOL" ]; then
if [ -f "pnpm-lock.yaml" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
if [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
else
CERC_BUILD_TOOL=npm
fi
fi
time $CERC_BUILD_TOOL install || exit 1
time $CERC_BUILD_TOOL build || exit 1
$CERC_BUILD_TOOL install || exit 1
$CERC_BUILD_TOOL build || exit 1
rm -rf "${DEST_DIR}"
if [ -z "${CERC_BUILD_OUTPUT_DIR}" ]; then
if [ -d "${WORK_DIR}/dist" ]; then
CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/dist"
elif [ -d "${WORK_DIR}/build" ]; then
CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/build"
else
echo "ERROR: Unable to locate build output. Set with --extra-build-args \"--build-arg CERC_BUILD_OUTPUT_DIR=path\"" 1>&2
exit 1
fi
fi
mv "${CERC_BUILD_OUTPUT_DIR}" "${DEST_DIR}"
mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
else
echo "Copying static app ..."
mv "${WORK_DIR}" "${DEST_DIR}"
@@ -1,118 +0,0 @@
# Ajna Watcher
## Setup
Clone required repositories:
```bash
laconic-so --stack ajna setup-repositories --git-ssh --pull
```
Build the container images:
```bash
laconic-so --stack ajna build-containers
```
## Deploy
Create a spec file for the deployment:
```bash
laconic-so --stack ajna deploy init --output ajna-spec.yml
```
### Ports
Edit `network` in the spec file to map container ports to host ports as required:
```yml
...
network:
ports:
ajna-watcher-db:
- 15432:5432
ajna-watcher-job-runner:
- 9000:9000
ajna-watcher-server:
- 3008:3008
- 9001:9001
```
### Create a deployment
Create a deployment from the spec file:
```bash
laconic-so --stack ajna deploy create --spec-file ajna-spec.yml --deployment-dir ajna-deployment
```
### Configuration
Inside deployment directory, open the `config.env` file and set following env variables:
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
```bash
laconic-so deployment --dir ajna-deployment start
```
* To list down and monitor the running containers:
```bash
# With status
docker ps -a
# Check logs for a container
docker logs -f <CONTAINER_ID>
```
* Open the GQL playground at <http://localhost:3008/graphql>
```graphql
# Example query
query {
_meta {
block {
hash
number
timestamp
}
deployment
hasIndexingErrors
}
accounts {
id
txCount
tokensDelegated
rewardsClaimed
}
}
```
## Clean up
Stop all the ajna services running in background:
```bash
# Only stop the docker containers
laconic-so deployment --dir ajna-deployment stop
# Run 'start' to restart the deployment
```
To stop all the ajna services and also delete data:
```bash
# Stop the docker containers
laconic-so deployment --dir ajna-deployment stop --delete-volumes
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r ajna-deployment
```
@@ -1,9 +0,0 @@
version: "1.0"
name: ajna
description: "Ajna watcher stack"
repos:
- git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
containers:
- cerc/watcher-ajna
pods:
- watcher-ajna
@@ -1,26 +0,0 @@
# Blast stack
## Clone required repositories
```
$ laconic-so --stack fixturenet-blast setup-repositories
```
## Build the stack's containers
```
$ laconic-so --stack fixturenet-blast build-containers
```
## Create a deployment of the stack
```
$ laconic-so --stack fixturenet-blast deploy init --map-ports-to-host any-same --output blast-spec.yml
```
[Insert details on how to configure the stack]
```
$ laconic-so --stack fixturenet-blast deploy create --deployment-dir blast-deployment --spec-file blast-spec.yml
```
## Start the stack
```
$ laconic-so deployment --dir blast-deployment start
```
Check logs:
```
$ laconic-so deployment --dir blast-deployment logs
```
@@ -1,17 +0,0 @@
version: "1.0"
name: fixturenet-blast
description: "A blast devnet stack"
repos:
- github.com/blast-io/blast
- git.vdb.to/cerc-io/lighthouse
containers:
- cerc/webapp-base
- cerc/lighthouse
- cerc/lighthouse-cli
- cerc/foundry
- cerc/fixturenet-eth-lighthouse
pods:
- fixturenet-blast
- foundry
@@ -1,26 +0,0 @@
# Blast stack
## Clone required repositories
```
$ laconic-so --stack mainnet-blast setup-repositories
```
## Build the stack's containers
```
$ laconic-so --stack mainnet-blast build-containers
```
## Create a deployment of the stack
```
$ laconic-so --stack mainnet-blast deploy init --map-ports-to-host any-same --output blast-spec.yml
```
[Insert details on how to configure the stack]
```
$ laconic-so --stack mainnet-blast deploy create --deployment-dir blast-deployment --spec-file blast-spec.yml
```
## Start the stack
```
$ laconic-so deployment --dir blast-deployment start
```
Check logs:
```
$ laconic-so deployment --dir blast-deployment logs
```
@@ -1,39 +0,0 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from pathlib import Path
from shutil import copy
import yaml
def create(context, extra_args):
# Our goal here is just to copy the json files for blast
yml_path = context.deployment_dir.joinpath("spec.yml")
with open(yml_path, 'r') as file:
data = yaml.safe_load(file)
mount_point = data['volumes']['blast-data']
if mount_point[0] == "/":
deploy_dir = Path(mount_point)
else:
deploy_dir = context.deployment_dir.joinpath(mount_point)
command_context = extra_args[2]
compose_file = [f for f in command_context.cluster_context.compose_files if "mainnet-blast" in f][0]
source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "genesis.json")
copy(source_config_file, deploy_dir)
source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "rollup.json")
copy(source_config_file, deploy_dir)
@@ -1,12 +0,0 @@
version: "1.0"
name: mainnet-blast
description: "A blast stack"
repos:
- github.com/blast-io/blast
- git.vdb.to/cerc-io/lighthouse
containers:
- cerc/webapp-base
- cerc/lighthouse
- cerc/lighthouse-cli
pods:
- mainnet-blast
@@ -16,55 +16,26 @@ laconic-so --stack merkl-sushiswap-v3 build-containers
## Deploy
Create a spec file for the deployment:
```bash
laconic-so --stack merkl-sushiswap-v3 deploy init --output merkl-sushiswap-v3-spec.yml
```
### Ports
Edit `network` in the spec file to map container ports to host ports as required:
```
...
network:
ports:
merkl-sushiswap-v3-watcher-db:
- '5432'
merkl-sushiswap-v3-watcher-job-runner:
- 9002:9000
merkl-sushiswap-v3-watcher-server:
- 127.0.0.1:3007:3008
- 9003:9001
```
### Create a deployment
Create a deployment from the spec file:
```bash
laconic-so --stack merkl-sushiswap-v3 deploy create --spec-file merkl-sushiswap-v3-spec.yml --deployment-dir merkl-sushiswap-v3-deployment
```
### Configuration
Inside deployment directory, open the `config.env` file and set following env variables:
Create and update an env file to be used in the next step:
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher
CERC_ETH_RPC_ENDPOINT=
```
### Deploy the stack
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
```bash
laconic-so deployment --dir merkl-sushiswap-v3-deployment start
laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
```
* To list down and monitor the running containers:
```bash
laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps
# With status
docker ps -a
@@ -75,7 +46,6 @@ laconic-so deployment --dir merkl-sushiswap-v3-deployment start
* Open the GQL playground at http://localhost:3007/graphql
```graphql
# Example query
{
_meta {
block {
@@ -84,7 +54,7 @@ laconic-so deployment --dir merkl-sushiswap-v3-deployment start
}
hasIndexingErrors
}
factories {
id
poolCount
@@ -94,21 +64,18 @@ laconic-so deployment --dir merkl-sushiswap-v3-deployment start
## Clean up
Stop all the merkl-sushiswap-v3 services running in background:
Stop all the services running in background:
```bash
# Only stop the docker containers
laconic-so deployment --dir merkl-sushiswap-v3-deployment stop
# Run 'start' to restart the deployment
laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down
```
To stop all the merkl-sushiswap-v3 services and also delete data:
Clear volumes created by this stack:
```bash
# Stop the docker containers
laconic-so deployment --dir merkl-sushiswap-v3-deployment stop --delete-volumes
# List all relevant volumes
docker volume ls -q --filter "name=merkl_sushiswap_v3"
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r merkl-sushiswap-v3-deployment
# Remove all the listed volumes
docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3")
```
@@ -2,7 +2,7 @@ version: "1.0"
name: merkl-sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6
containers:
- cerc/watcher-merkl-sushiswap-v3
pods:
@@ -1,7 +1,7 @@
version: "0.1"
name: monitoring
repos:
- github.com/cerc-io/watcher-ts@v0.2.81
- github.com/cerc-io/watcher-ts@v0.2.79
containers:
- cerc/watcher-ts
pods:
@@ -55,7 +55,7 @@ ports:
Create deployment:
```bash
laconic-so --stack sushiswap-subgraph deploy create --spec-file sushiswap-subgraph-spec.yml --deployment-dir sushiswap-subgraph-deployment
laconic-so deploy create --spec-file sushiswap-subgraph-spec.yml --deployment-dir sushiswap-subgraph-deployment
```
## Start the stack
@@ -16,55 +16,26 @@ laconic-so --stack sushiswap-v3 build-containers
## Deploy
Create a spec file for the deployment:
```bash
laconic-so --stack sushiswap-v3 deploy init --output sushiswap-v3-spec.yml
```
### Ports
Edit `network` in the spec file to map container ports to host ports as required:
```
...
network:
ports:
sushiswap-v3-watcher-db:
- '5432'
sushiswap-v3-watcher-job-runner:
- 9000:9000
sushiswap-v3-watcher-server:
- 127.0.0.1:3008:3008
- 9001:9001
```
### Create a deployment
Create a deployment from the spec file:
```bash
laconic-so --stack sushiswap-v3 deploy create --spec-file sushiswap-v3-spec.yml --deployment-dir sushiswap-v3-deployment
```
### Configuration
Inside deployment directory, open the `config.env` file and set following env variables:
Create and update an env file to be used in the next step:
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher
CERC_ETH_RPC_ENDPOINT=
```
### Deploy the stack
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
```bash
laconic-so deployment --dir sushiswap-v3-deployment start
laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
```
* To list down and monitor the running containers:
```bash
laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 ps
# With status
docker ps -a
@@ -72,43 +43,20 @@ laconic-so deployment --dir sushiswap-v3-deployment start
docker logs -f <CONTAINER_ID>
```
* Open the GQL playground at http://localhost:3008/graphql
```graphql
# Example query
{
_meta {
block {
number
timestamp
}
hasIndexingErrors
}
factories {
id
poolCount
}
}
```
## Clean up
Stop all the sushiswap-v3 services running in background:
Stop all the services running in background:
```bash
# Only stop the docker containers
laconic-so deployment --dir sushiswap-v3-deployment stop
# Run 'start' to restart the deployment
laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 down
```
To stop all the sushiswap-v3 services and also delete data:
Clear volumes created by this stack:
```bash
# Stop the docker containers
laconic-so deployment --dir sushiswap-v3-deployment stop --delete-volumes
# List all relevant volumes
docker volume ls -q --filter "name=sushiswap_v3"
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r sushiswap-v3-deployment
# Remove all the listed volumes
docker volume rm $(docker volume ls -q --filter "name=sushiswap_v3")
```
@@ -2,7 +2,7 @@ version: "1.0"
name: sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.7
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.6
containers:
- cerc/watcher-sushiswap-v3
pods:
@@ -33,29 +33,23 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec
### Ports
Edit `uniswap-urbit-app-spec.yml` such that it looks like:
Edit `network` in spec file to map container ports to same ports in host:
```yml
stack: uniswap-urbit-app
deploy-to: compose
...
network:
ports:
proxy-server:
- '4000:4000'
urbit-fake-ship:
- '8080:80'
proxy-server:
- '4000:4000'
ipfs:
- '4001'
- '8081:8080'
- 0.0.0.0:5001:5001
volumes:
urbit_app_builds: ./data/urbit_app_builds
urbit_data: ./data/urbit_data
ipfs-import: ./data/ipfs-import
ipfs-data: ./data/ipfs-data
- '8081:8080'
- '5001:5001'
...
```
Note: Skip the `ipfs` ports if using an externally running IPFS node, set via `config.env`, below.
Note: Skip the `ipfs` ports if need to use an externally running IPFS node
### Data volumes
-23
View File
@@ -29,29 +29,6 @@ def _image_needs_pushed(image: str):
return image.endswith(":local")
def remote_image_exists(remote_repo_url: str, local_tag: str):
docker = DockerClient()
try:
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
result = docker.manifest.inspect(remote_tag)
return True if result else False
except Exception: # noqa: E722
return False
def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
if not additional_tags:
return
if not remote_image_exists(remote_repo_url, local_tag):
raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
docker = DockerClient()
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
def remote_tag_for_image(image: str, remote_repo_url: str):
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
major_parts = image.split("/", 2)
+7 -11
View File
@@ -101,7 +101,7 @@ class ClusterInfo:
)
return service
def get_ingress(self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"):
def get_ingress(self, use_tls=False):
# No ingress for a deployment that has no http-proxy defined, for now
http_proxy_info_list = self.spec.get_http_proxy()
ingress = None
@@ -114,8 +114,8 @@ class ClusterInfo:
host_name = http_proxy_info["host-name"]
rules = []
tls = [client.V1IngressTLS(
hosts=certificate["spec"]["dnsNames"] if certificate else [host_name],
secret_name=certificate["spec"]["secretName"] if certificate else f"{self.app_name}-tls"
hosts=[host_name],
secret_name=f"{self.app_name}-tls"
)] if use_tls else None
paths = []
for route in http_proxy_info["routes"]:
@@ -147,17 +147,13 @@ class ClusterInfo:
tls=tls,
rules=rules
)
ingress_annotations = {
"kubernetes.io/ingress.class": "nginx",
}
if not certificate:
ingress_annotations["cert-manager.io/cluster-issuer"] = cluster_issuer
ingress = client.V1Ingress(
metadata=client.V1ObjectMeta(
name=f"{self.app_name}-ingress",
annotations=ingress_annotations
annotations={
"kubernetes.io/ingress.class": "nginx",
"cert-manager.io/cluster-issuer": "letsencrypt-prod"
}
),
spec=spec
)
+3 -45
View File
@@ -169,39 +169,6 @@ class K8sDeployer(Deployer):
print("Service created:")
print(f"{service_resp}")
def _find_certificate_for_host_name(self, host_name):
all_certificates = self.custom_obj_api.list_namespaced_custom_object(
group="cert-manager.io",
version="v1",
namespace=self.k8s_namespace,
plural="certificates"
)
host_parts = host_name.split(".", 1)
host_as_wild = None
if len(host_parts) == 2:
host_as_wild = f"*.{host_parts[1]}"
now = datetime.utcnow().replace(tzinfo=timezone.utc)
fmt = "%Y-%m-%dT%H:%M:%S%z"
# Walk over all the configured certificates.
for cert in all_certificates["items"]:
dns = cert["spec"]["dnsNames"]
# Check for an exact hostname match or a wildcard match.
if host_name in dns or host_as_wild in dns:
status = cert.get("status", {})
# Check the certificate date.
if "notAfter" in status and "notBefore" in status:
before = datetime.strptime(status["notBefore"], fmt)
after = datetime.strptime(status["notAfter"], fmt)
if before < now < after:
# Check the status is Ready
for condition in status.get("conditions", []):
if "True" == condition.get("status") and "Ready" == condition.get("type"):
return cert
return None
def up(self, detach, services):
if not opts.o.dry_run:
if self.is_kind():
@@ -222,15 +189,8 @@ class K8sDeployer(Deployer):
self._create_volume_data()
self._create_deployment()
http_proxy_info = self.cluster_info.spec.get_http_proxy()
# Note: at present we don't support tls for kind (and enabling tls causes errors)
use_tls = http_proxy_info and not self.is_kind()
certificate = self._find_certificate_for_host_name(http_proxy_info[0]["host-name"]) if use_tls else None
if opts.o.debug:
if certificate:
print(f"Using existing certificate: {certificate}")
ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=use_tls, certificate=certificate)
ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
if ingress:
if opts.o.debug:
print(f"Sending this ingress: {ingress}")
@@ -390,11 +350,9 @@ class K8sDeployer(Deployer):
name=ingress.spec.tls[0].secret_name
)
hostname = ingress.spec.rules[0].host
hostname = ingress.spec.tls[0].hosts[0]
ip = ingress.status.load_balancer.ingress[0].ip
tls = "notBefore: %s; notAfter: %s; names: %s" % (
cert["status"]["notBefore"], cert["status"]["notAfter"], ingress.spec.tls[0].hosts
)
tls = "notBefore: %s, notAfter: %s" % (cert["status"]["notBefore"], cert["status"]["notAfter"])
except: # noqa: E722
pass
+24 -22
View File
@@ -27,9 +27,7 @@ class ResourceLimits:
memory: int = None
storage: int = None
def __init__(self, obj=None):
if obj is None:
obj = {}
def __init__(self, obj={}):
if "cpus" in obj:
self.cpus = float(obj["cpus"])
if "memory" in obj:
@@ -52,9 +50,7 @@ class Resources:
limits: ResourceLimits = None
reservations: ResourceLimits = None
def __init__(self, obj=None):
if obj is None:
obj = {}
def __init__(self, obj={}):
if "reservations" in obj:
self.reservations = ResourceLimits(obj["reservations"])
if "limits" in obj:
@@ -76,9 +72,7 @@ class Spec:
obj: typing.Any
file_path: Path
def __init__(self, file_path: Path = None, obj=None) -> None:
if obj is None:
obj = {}
def __init__(self, file_path: Path = None, obj={}) -> None:
self.file_path = file_path
self.obj = obj
@@ -97,41 +91,49 @@ class Spec:
self.file_path = file_path
def get_image_registry(self):
return self.obj.get(constants.image_registry_key)
return (self.obj[constants.image_registry_key]
if self.obj and constants.image_registry_key in self.obj
else None)
def get_volumes(self):
return self.obj.get(constants.volumes_key, {})
return (self.obj["volumes"]
if self.obj and "volumes" in self.obj
else {})
def get_configmaps(self):
return self.obj.get(constants.configmaps_key, {})
return (self.obj["configmaps"]
if self.obj and "configmaps" in self.obj
else {})
def get_container_resources(self):
return Resources(self.obj.get(constants.resources_key, {}).get("containers", {}))
return Resources(self.obj.get("resources", {}).get("containers", {}))
def get_volume_resources(self):
return Resources(self.obj.get(constants.resources_key, {}).get(constants.volumes_key, {}))
return Resources(self.obj.get("resources", {}).get("volumes", {}))
def get_http_proxy(self):
return self.obj.get(constants.network_key, {}).get(constants.http_proxy_key, [])
return (self.obj[constants.network_key][constants.http_proxy_key]
if self.obj and constants.network_key in self.obj
and constants.http_proxy_key in self.obj[constants.network_key]
else None)
def get_annotations(self):
return self.obj.get(constants.annotations_key, {})
return self.obj.get("annotations", {})
def get_labels(self):
return self.obj.get(constants.labels_key, {})
return self.obj.get("labels", {})
def get_privileged(self):
return "true" == str(self.obj.get(constants.security_key, {}).get("privileged", "false")).lower()
return "true" == str(self.obj.get("security", {}).get("privileged", "false")).lower()
def get_capabilities(self):
return self.obj.get(constants.security_key, {}).get("capabilities", [])
return self.obj.get("security", {}).get("capabilities", [])
def get_deployment_type(self):
return self.obj.get(constants.deploy_to_key)
return self.obj[constants.deploy_to_key]
def is_kubernetes_deployment(self):
return self.get_deployment_type() in [constants.k8s_kind_deploy_type,
constants.k8s_deploy_type]
return self.get_deployment_type() in [constants.k8s_kind_deploy_type, constants.k8s_deploy_type]
def is_kind_deployment(self):
return self.get_deployment_type() in [constants.k8s_kind_deploy_type]
@@ -24,9 +24,8 @@ import uuid
import click
from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
from stack_orchestrator.deploy.webapp import deploy_webapp
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedLogger,
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
build_container_image, push_container_image,
file_hash, deploy_to_k8s, publish_deployment,
hostname_for_deployment_request, generate_hostname_for_app,
@@ -34,6 +33,7 @@ from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedL
def process_app_deployment_request(
run_id,
ctx,
laconic: LaconicRegistryClient,
app_deployment_request,
@@ -43,18 +43,13 @@ def process_app_deployment_request(
deployment_parent_dir,
kube_config,
image_registry,
force_rebuild,
logger
log_file=None
):
logger.log("BEGIN - process_app_deployment_request")
# 1. look up application
app = laconic.get_record(app_deployment_request.attributes.application, require=True)
logger.log(f"Retrieved app record {app_deployment_request.attributes.application}")
# 2. determine dns
requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
logger.log(f"Determined requested name: {requested_name}")
# HACK
if "." in requested_name:
@@ -72,7 +67,7 @@ def process_app_deployment_request(
matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True))
if matched_owner:
logger.log(f"Matched DnsRecord ownership: {matched_owner}")
print("Matched DnsRecord ownership:", matched_owner)
else:
raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
(dns_record.id, app_deployment_request.id))
@@ -96,9 +91,7 @@ def process_app_deployment_request(
deployment_record = laconic.get_record(app_deployment_crn)
deployment_dir = os.path.join(deployment_parent_dir, fqdn)
deployment_config_file = os.path.join(deployment_dir, "config.env")
# TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
app_image_shared_tag = f"laconic-webapp/{app.id}:local"
# b. check for deployment directory (create if necessary)
if not os.path.exists(deployment_dir):
if deployment_record:
@@ -113,26 +106,11 @@ def process_app_deployment_request(
needs_k8s_deploy = False
# 6. build container (if needed)
if not deployment_record or deployment_record.attributes.application != app.id:
# TODO: pull from request
extra_build_args = []
build_container_image(app, deployment_container_tag, extra_build_args, log_file)
push_container_image(deployment_dir, log_file)
needs_k8s_deploy = True
# check if the image already exists
shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
if shared_tag_exists and not force_rebuild:
# simply add our unique tag to the existing image and we are done
logger.log(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}")
add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
logger.log("Tag complete")
else:
extra_build_args = [] # TODO: pull from request
logger.log(f"Building container image {deployment_container_tag}")
build_container_image(app, deployment_container_tag, extra_build_args, logger)
logger.log("Build complete")
logger.log(f"Pushing container image {deployment_container_tag}")
push_container_image(deployment_dir, logger)
logger.log("Push complete")
# The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
logger.log(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}")
add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
logger.log("Tag complete")
# 7. update config (if needed)
if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
@@ -140,13 +118,13 @@ def process_app_deployment_request(
# 8. update k8s deployment
if needs_k8s_deploy:
print("Deploying to k8s")
deploy_to_k8s(
deployment_record,
deployment_dir,
logger
log_file
)
logger.log("Publishing deployment to registry.")
publish_deployment(
laconic,
app,
@@ -155,11 +133,8 @@ def process_app_deployment_request(
dns_record,
dns_crn,
deployment_dir,
app_deployment_request,
logger
app_deployment_request
)
logger.log("Publication complete.")
logger.log("END - process_app_deployment_request")
def load_known_requests(filename):
@@ -196,13 +171,12 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
@click.pass_context
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901
request_id, discover, state_file, only_update_state,
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
include_tags, exclude_tags, force_rebuild, log_dir):
include_tags, exclude_tags, log_dir):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
sys.exit(2)
@@ -321,9 +295,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
run_log_file = open(run_log_file_path, "wt")
run_reg_client = LaconicRegistryClient(laconic_config, log_file=run_log_file)
logger = TimedLogger(run_id, run_log_file)
logger.log("Processing ...")
process_app_deployment_request(
run_id,
ctx,
run_reg_client,
r,
@@ -333,15 +306,12 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
os.path.abspath(deployment_parent_dir),
kube_config,
image_registry,
force_rebuild,
logger
run_log_file
)
status = "DEPLOYED"
except Exception as e:
logger.log("ERROR: " + str(e))
print("ERROR: " + str(e), file=run_log_file)
finally:
if logger:
logger.log(f"DONE with status {status}", show_step_time=False, show_total_time=True)
dump_known_requests(state_file, [r], status)
if run_log_file:
run_log_file.close()
+17 -47
View File
@@ -13,7 +13,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import datetime
import hashlib
import json
import os
@@ -40,25 +39,6 @@ class AttrDict(dict):
return v
class TimedLogger:
def __init__(self, id="", file=None):
self.start = datetime.datetime.now()
self.last = self.start
self.id = id
self.file = file
def log(self, msg, show_step_time=True, show_total_time=False):
prefix = f"{datetime.datetime.utcnow()} - {self.id}"
if show_step_time:
prefix += f" - {datetime.datetime.now() - self.last} (step)"
if show_total_time:
prefix += f" - {datetime.datetime.now() - self.start} (total)"
print(f"{prefix}: {msg}", file=self.file)
if self.file:
self.file.flush()
self.last = datetime.datetime.now()
def logged_cmd(log_file, *vargs):
result = None
try:
@@ -239,7 +219,7 @@ def determine_base_container(clone_dir, app_type="webapp"):
return base_container
def build_container_image(app_record, tag, extra_build_args=[], logger=None):
def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
tmpdir = tempfile.mkdtemp()
try:
@@ -248,33 +228,31 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
repo = random.choice(app_record.attributes.repository)
clone_dir = os.path.join(tmpdir, record_id)
logger.log(f"Cloning repository {repo} to {clone_dir} ...")
print(f"Cloning repository {repo} to {clone_dir} ...")
if ref:
# TODO: Determing branch or hash, and use depth 1 if we can.
git_env = dict(os.environ.copy())
# Never prompt
git_env["GIT_TERMINAL_PROMPT"] = "0"
try:
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=logger.file, stderr=logger.file)
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=log_file, stderr=log_file)
except Exception as e:
logger.log(f"git clone failed. Is the repository {repo} private?")
print(f"git clone failed. Is the repository {repo} private?", file=log_file)
raise e
try:
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=logger.file, stderr=logger.file)
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=log_file, stderr=log_file)
except Exception as e:
logger.log(f"git checkout failed. Does ref {ref} exist?")
print(f"git checkout failed. Does ref {ref} exist?", file=log_file)
raise e
else:
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=logger.file, stderr=logger.file)
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=log_file, stderr=log_file)
result.check_returncode()
base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
logger.log("Building webapp ...")
print("Building webapp ...")
build_command = [
sys.argv[0],
"--verbose",
"build-webapp",
sys.argv[0], "build-webapp",
"--source-repo", clone_dir,
"--tag", tag,
"--base-container", base_container
@@ -283,31 +261,28 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
build_command.append("--extra-build-args")
build_command.append(" ".join(extra_build_args))
result = subprocess.run(build_command, stdout=logger.file, stderr=logger.file)
result = subprocess.run(build_command, stdout=log_file, stderr=log_file)
result.check_returncode()
finally:
logged_cmd(logger.file, "rm", "-rf", tmpdir)
logged_cmd(log_file, "rm", "-rf", tmpdir)
def push_container_image(deployment_dir, logger):
logger.log("Pushing images ...")
def push_container_image(deployment_dir, log_file=None):
print("Pushing image ...")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"],
stdout=logger.file, stderr=logger.file)
stdout=log_file, stderr=log_file)
result.check_returncode()
logger.log("Finished pushing images.")
def deploy_to_k8s(deploy_record, deployment_dir, logger):
def deploy_to_k8s(deploy_record, deployment_dir, log_file=None):
if not deploy_record:
command = "up"
else:
command = "update"
logger.log("Deploying to k8s ...")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
stdout=logger.file, stderr=logger.file)
stdout=log_file, stderr=log_file)
result.check_returncode()
logger.log("Finished deploying to k8s.")
def publish_deployment(laconic: LaconicRegistryClient,
@@ -317,8 +292,7 @@ def publish_deployment(laconic: LaconicRegistryClient,
dns_record,
dns_crn,
deployment_dir,
app_deployment_request=None,
logger=None):
app_deployment_request=None):
if not deploy_record:
deploy_ver = "0.0.1"
else:
@@ -348,8 +322,6 @@ def publish_deployment(laconic: LaconicRegistryClient,
if app_deployment_request:
new_dns_record["record"]["request"] = app_deployment_request.id
if logger:
logger.log("Publishing DnsRecord.")
dns_id = laconic.publish(new_dns_record, [dns_crn])
new_deployment_record = {
@@ -369,8 +341,6 @@ def publish_deployment(laconic: LaconicRegistryClient,
if app_deployment_request:
new_deployment_record["record"]["request"] = app_deployment_request.id
if logger:
logger.log("Publishing ApplicationDeploymentRecord.")
deployment_id = laconic.publish(new_deployment_record, [deployment_crn])
return {"dns": dns_id, "deployment": deployment_id}
+1 -2
View File
@@ -17,7 +17,7 @@ import click
from stack_orchestrator.command_types import CommandOptions
from stack_orchestrator.repos import setup_repositories
from stack_orchestrator.build import build_containers, fetch_containers
from stack_orchestrator.build import build_containers
from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy.webapp import (run_webapp,
@@ -52,7 +52,6 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
cli.add_command(setup_repositories.command, "setup-repositories")
cli.add_command(build_containers.command, "build-containers")
cli.add_command(fetch_containers.command, "fetch-containers")
cli.add_command(build_npms.command, "build-npms")
cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(run_webapp.command, "run-webapp")
-43
View File
@@ -1,43 +0,0 @@
#!/usr/bin/env bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Laconic registry CLI tests"
env
cat /etc/hosts
# Bit of a hack, test the most recent package
TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
echo "$(date +"%Y-%m-%d %T"): Cloning laconic-registry-cli repository into: $CERC_REPO_BASE_DIR"
$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories --include git.vdb.to/cerc-io/laconic-registry-cli
echo "$(date +"%Y-%m-%d %T"): Starting stack"
TEST_AUCTION_ENABLED=true BASE_DIR=${CERC_REPO_BASE_DIR} $TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd up
echo "$(date +"%Y-%m-%d %T"): Stack started"
# Verify that the fixturenet is up and running
$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps
# Get the fixturenet account address
laconicd_account_address=$(docker exec laconicd-laconicd-1 laconicd keys list | awk '/- address:/ {print $3}')
# Copy over config
docker exec laconicd-cli-1 cp config.yml laconic-registry-cli/
# Wait for the laconid endpoint to come up
echo "Waiting for the RPC endpoint to come up"
docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
# Run the tests
echo "Running the tests"
docker exec -e TEST_ACCOUNT=$laconicd_account_address laconicd-cli-1 sh -c 'cd laconic-registry-cli && yarn && yarn test'
# Clean up
$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd down --delete-volumes
echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
rm -rf $CERC_REPO_BASE_DIR
echo "$(date +"%Y-%m-%d %T"): Test finished"
-1
View File
@@ -12,7 +12,6 @@ cat /etc/hosts
TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
# Set a new unique repo dir
export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO"
echo "$(date +"%Y-%m-%d %T"): Test version command"
reported_version_string=$( $TEST_TARGET_SO version )
+2 -2
View File
@@ -32,14 +32,14 @@ set +e
CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
sleep 3
wget --tries 20 --retry-connrefused --waitretry=3 -O test.before -m http://localhost:3000
wget -t 7 -O test.before -m http://localhost:3000
docker logs $CONTAINER_ID
docker remove -f $CONTAINER_ID
CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
sleep 3
wget --tries 20 --retry-connrefused --waitretry=3 -O test.after -m http://localhost:3000
wget -t 7 -O test.after -m http://localhost:3000
docker logs $CONTAINER_ID
docker remove -f $CONTAINER_ID