Compare commits

..
Author SHA1 Message Date
iskay f088cbb3b0 fix linter errors 2023-11-14 14:38:49 +00:00
iskay 3db443b2bb fix commands import 2023-11-10 20:42:02 +00:00
iskay 1072fc98c3 update fixturenet-optimism 2023-11-10 20:05:22 +00:00
telackey 042b413598 Support the case where webpack config is already present next.config.js (#631)
* Support the case where webpack config is already present next.config.js

* Update scripts for experimental-compile/experimental-generate
2023-11-08 23:44:48 -06:00
dboreham 8384e95049 Add debug commands to test job (#636) 2023-11-08 19:42:19 -07:00
dboreham a27cf86748 Add basic k8s test (#635)
* Add CI job

* Add basic k8s test
2023-11-08 19:12:48 -07:00
dboreham ce587457d7 Add env var support for k8s (#634) 2023-11-08 17:53:46 -07:00
dboreham 5e91c2224e kind test stack (#629) 2023-11-08 01:11:00 -07:00
telackey 36e13f7199 Remove test output from tree. (#628) 2023-11-07 23:10:32 -06:00
telackey d9bcc088a8 Enable webapp test in GitHub CI. (#627) 2023-11-07 18:27:08 -06:00
telackey 660326f713 Add new build-webapp command and related scripts and containers. (#626)
* Add new build-webapp command and related scripts and containers.
2023-11-07 18:15:04 -06:00
45 changed files with 1511 additions and 562 deletions
+55
View File
@@ -0,0 +1,55 @@
name: K8s Deploy Test
on:
pull_request:
branches: '*'
push:
branches:
- main
- ci-test
paths-ignore:
- '.gitea/workflows/triggers/*'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
name: "Run deploy test suite"
runs-on: ubuntu-latest
steps:
- name: "Clone project repository"
uses: actions/checkout@v3
# At present the stock setup-python action fails on Linux/aarch64
# Conditional steps below workaroud this by using deadsnakes for that case only
- name: "Install Python for ARM on Linux"
if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
uses: deadsnakes/action@v3.0.1
with:
python-version: '3.8'
- name: "Install Python cases other than ARM on Linux"
if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: "Install Go"
uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: "Install Kind"
run: go install sigs.k8s.io/kind@v0.20.0
- name: "Debug Kind"
run: kind create cluster --retain && docker logs kind-control-plane
+49
View File
@@ -0,0 +1,49 @@
name: Webapp Test
on:
pull_request:
branches: '*'
push:
branches:
- main
- ci-test
paths-ignore:
- '.gitea/workflows/triggers/*'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
name: "Run webapp test suite"
runs-on: ubuntu-latest
steps:
- name: "Clone project repository"
uses: actions/checkout@v3
# At present the stock setup-python action fails on Linux/aarch64
# Conditional steps below workaroud this by using deadsnakes for that case only
- name: "Install Python for ARM on Linux"
if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
uses: deadsnakes/action@v3.0.1
with:
python-version: '3.8'
- name: "Install Python cases other than ARM on Linux"
if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: "Run webapp tests"
run: ./tests/webapp-test/run-webapp-test.sh
+29
View File
@@ -0,0 +1,29 @@
name: Webapp Test
on:
pull_request:
branches: '*'
push:
branches: '*'
jobs:
test:
name: "Run webapp test suite"
runs-on: ubuntu-latest
steps:
- name: "Clone project repository"
uses: actions/checkout@v3
- name: "Install Python"
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: "Run webapp tests"
run: ./tests/webapp-test/run-webapp-test.sh
+1
View File
@@ -1,4 +1,5 @@
python-decouple>=3.8
python-dotenv==1.0.0
GitPython>=3.1.32
tqdm>=4.65.0
python-on-whales>=0.64.0
+74 -52
View File
@@ -33,6 +33,73 @@ from stack_orchestrator.base import get_npm_registry_url
# TODO: find a place for this
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
def make_container_build_env(dev_root_path: str,
container_build_dir: str,
debug: bool,
force_rebuild: bool,
extra_build_args: str):
container_build_env = {
"CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
"CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""),
"CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""),
"CERC_REPO_BASE_DIR": dev_root_path,
"CERC_CONTAINER_BASE_DIR": container_build_dir,
"CERC_HOST_UID": f"{os.getuid()}",
"CERC_HOST_GID": f"{os.getgid()}",
"DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0")
}
container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {})
docker_host_env = os.getenv("DOCKER_HOST")
if docker_host_env:
container_build_env.update({"DOCKER_HOST": docker_host_env})
return container_build_env
def process_container(container,
container_build_dir: str,
container_build_env: dict,
dev_root_path: str,
quiet: bool,
verbose: bool,
dry_run: bool,
continue_on_error: bool,
):
if not quiet:
print(f"Building: {container}")
build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
build_script_filename = os.path.join(build_dir, "build.sh")
if verbose:
print(f"Build script filename: {build_script_filename}")
if os.path.exists(build_script_filename):
build_command = build_script_filename
else:
if verbose:
print(f"No script file found: {build_script_filename}, using default build script")
repo_dir = container.split('/')[1]
# TODO: make this less of a hack -- should be specified in some metadata somewhere
# Check if we have a repo for this container. If not, set the context dir to the container-build subdir
repo_full_path = os.path.join(dev_root_path, repo_dir)
repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
build_command = os.path.join(container_build_dir,
"default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
if not dry_run:
if verbose:
print(f"Executing: {build_command} with environment: {container_build_env}")
build_result = subprocess.run(build_command, shell=True, env=container_build_env)
if verbose:
print(f"Return code is: {build_result.returncode}")
if build_result.returncode != 0:
print(f"Error running build for {container}")
if not continue_on_error:
print("FATAL Error: container build failed and --continue-on-error not set, exiting")
sys.exit(1)
else:
print("****** Container Build Error, continuing because --continue-on-error is set")
else:
print("Skipped")
@click.command()
@click.option('--include', help="only build these containers")
@@ -83,61 +150,16 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
if stack:
print(f"Stack: {stack}")
# TODO: make this configurable
container_build_env = {
"CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
"CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""),
"CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""),
"CERC_REPO_BASE_DIR": dev_root_path,
"CERC_CONTAINER_BASE_DIR": container_build_dir,
"CERC_HOST_UID": f"{os.getuid()}",
"CERC_HOST_GID": f"{os.getgid()}",
"DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0")
}
container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {})
docker_host_env = os.getenv("DOCKER_HOST")
if docker_host_env:
container_build_env.update({"DOCKER_HOST": docker_host_env})
def process_container(container):
if not quiet:
print(f"Building: {container}")
build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
build_script_filename = os.path.join(build_dir, "build.sh")
if verbose:
print(f"Build script filename: {build_script_filename}")
if os.path.exists(build_script_filename):
build_command = build_script_filename
else:
if verbose:
print(f"No script file found: {build_script_filename}, using default build script")
repo_dir = container.split('/')[1]
# TODO: make this less of a hack -- should be specified in some metadata somewhere
# Check if we have a repo for this container. If not, set the context dir to the container-build subdir
repo_full_path = os.path.join(dev_root_path, repo_dir)
repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
build_command = os.path.join(container_build_dir, "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
if not dry_run:
if verbose:
print(f"Executing: {build_command} with environment: {container_build_env}")
build_result = subprocess.run(build_command, shell=True, env=container_build_env)
if verbose:
print(f"Return code is: {build_result.returncode}")
if build_result.returncode != 0:
print(f"Error running build for {container}")
if not continue_on_error:
print("FATAL Error: container build failed and --continue-on-error not set, exiting")
sys.exit(1)
else:
print("****** Container Build Error, continuing because --continue-on-error is set")
else:
print("Skipped")
container_build_env = make_container_build_env(dev_root_path,
container_build_dir,
debug,
force_rebuild,
extra_build_args)
for container in containers_in_scope:
if include_exclude_check(container, include, exclude):
process_container(container)
process_container(container, container_build_dir, container_build_env,
dev_root_path, quiet, verbose, dry_run, continue_on_error)
else:
if verbose:
print(f"Excluding: {container}")
+77
View File
@@ -0,0 +1,77 @@
# Copyright © 2022, 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
# Builds webapp containers
# env vars:
# CERC_REPO_BASE_DIR defaults to ~/cerc
# TODO: display the available list of containers; allow re-build of either all or specific containers
import os
from decouple import config
import click
from pathlib import Path
from stack_orchestrator.build import build_containers
@click.command()
@click.option('--base-container', default="cerc/nextjs-base")
@click.option('--source-repo', help="directory containing the webapp to build", required=True)
@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
@click.option("--extra-build-args", help="Supply extra arguments to build")
@click.pass_context
def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
'''build the specified webapp container'''
quiet = ctx.obj.quiet
verbose = ctx.obj.verbose
dry_run = ctx.obj.dry_run
debug = ctx.obj.debug
local_stack = ctx.obj.local_stack
stack = ctx.obj.stack
continue_on_error = ctx.obj.continue_on_error
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
if local_stack:
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
else:
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
if not quiet:
print(f'Dev Root is: {dev_root_path}')
# First build the base container.
container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
force_rebuild, extra_build_args)
build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
verbose, dry_run, continue_on_error)
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
base_container.replace("/", "-"),
"Dockerfile.webapp")
webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
verbose, dry_run, continue_on_error)
@@ -6,8 +6,8 @@ services:
# Deploys the L1 smart contracts (outputs to volume l1_deployment)
fixturenet-optimism-contracts:
restart: on-failure
hostname: fixturenet-optimism-contracts
image: cerc/optimism-contracts:local
hostname: fixturenet-optimism-contracts
env_file:
- ../config/fixturenet-optimism/l1-params.env
environment:
@@ -17,27 +17,49 @@ services:
CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL}
CERC_L1_ADDRESS: ${CERC_L1_ADDRESS}
CERC_L1_PRIV_KEY: ${CERC_L1_PRIV_KEY}
CERC_L1_ADDRESS_2: ${CERC_L1_ADDRESS_2}
CERC_L1_PRIV_KEY_2: ${CERC_L1_PRIV_KEY_2}
# Waits for L1 endpoint to be up before running the script
command: |
"./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./run.sh"
volumes:
- ../config/network/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh
- ../config/optimism-contracts/hardhat-tasks/verify-contract-deployment.ts:/app/packages/contracts-bedrock/tasks/verify-contract-deployment.ts
- ../config/optimism-contracts/hardhat-tasks/rekey-json.ts:/app/packages/contracts-bedrock/tasks/rekey-json.ts
- ../config/optimism-contracts/hardhat-tasks/send-balance.ts:/app/packages/contracts-bedrock/tasks/send-balance.ts
- ../config/fixturenet-optimism/optimism-contracts/update-config.js:/app/packages/contracts-bedrock/update-config.js
- ../config/fixturenet-optimism/optimism-contracts/run.sh:/app/packages/contracts-bedrock/run.sh
- ../config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh:/app/packages/contracts-bedrock/deploy-contracts.sh
- l2_accounts:/l2-accounts
- l1_deployment:/app/packages/contracts-bedrock
- l1_deployment:/l1-deployment
- l2_config:/l2-config
# Waits for L1 endpoint to be up before running the contract deploy script
command: |
"./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./deploy-contracts.sh"
# Initializes and runs the L2 execution client (outputs to volume l2_geth_data)
op-geth:
restart: always
image: cerc/optimism-l2geth:local
hostname: op-geth
depends_on:
op-node:
condition: service_started
volumes:
- ../config/fixturenet-optimism/run-op-geth.sh:/run-op-geth.sh
- l2_config:/l2-config:ro
- l2_accounts:/l2-accounts:ro
- l2_geth_data:/datadir
entrypoint: "sh"
command: "/run-op-geth.sh"
ports:
- "8545"
- "8546"
healthcheck:
test: ["CMD", "nc", "-vz", "localhost:8545"]
interval: 30s
timeout: 10s
retries: 100
start_period: 10s
extra_hosts:
- "host.docker.internal:host-gateway"
# Generates the config files required for L2 (outputs to volume l2_config)
op-node-l2-config-gen:
restart: on-failure
# Runs the L2 consensus client (Sequencer node)
# Generates the L2 config files if not already present (outputs to volume l2_config)
op-node:
restart: always
image: cerc/optimism-op-node:local
hostname: op-node
depends_on:
fixturenet-optimism-contracts:
condition: service_completed_successfully
@@ -47,61 +69,19 @@ services:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_L1_RPC: ${CERC_L1_RPC}
volumes:
- ../config/fixturenet-optimism/generate-l2-config.sh:/app/generate-l2-config.sh
- l1_deployment:/contracts-bedrock:ro
- l2_config:/app
command: ["sh", "/app/generate-l2-config.sh"]
extra_hosts:
- "host.docker.internal:host-gateway"
# Initializes and runs the L2 execution client (outputs to volume l2_geth_data)
op-geth:
restart: always
image: cerc/optimism-l2geth:local
depends_on:
op-node-l2-config-gen:
condition: service_started
volumes:
- ../config/fixturenet-optimism/run-op-geth.sh:/run-op-geth.sh
- l2_config:/op-node:ro
- ../config/fixturenet-optimism/run-op-node.sh:/run-op-node.sh
- l1_deployment:/l1-deployment:ro
- l2_config:/l2-config
- l2_accounts:/l2-accounts:ro
- l2_geth_data:/datadir
entrypoint: "sh"
command: "/run-op-geth.sh"
command: "/run-op-node.sh"
ports:
- "0.0.0.0:8545:8545"
- "0.0.0.0:8546:8546"
healthcheck:
test: ["CMD", "nc", "-vz", "localhost:8545"]
interval: 30s
timeout: 10s
retries: 10
start_period: 10s
# Runs the L2 consensus client (Sequencer node)
op-node:
restart: always
image: cerc/optimism-op-node:local
depends_on:
op-geth:
condition: service_healthy
env_file:
- ../config/fixturenet-optimism/l1-params.env
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_L1_RPC: ${CERC_L1_RPC}
volumes:
- ../config/fixturenet-optimism/run-op-node.sh:/app/run-op-node.sh
- l2_config:/op-node-data:ro
- l2_accounts:/l2-accounts:ro
command: ["sh", "/app/run-op-node.sh"]
ports:
- "0.0.0.0:8547:8547"
- "8547"
healthcheck:
test: ["CMD", "nc", "-vz", "localhost:8547"]
interval: 30s
timeout: 10s
retries: 10
retries: 100
start_period: 10s
extra_hosts:
- "host.docker.internal:host-gateway"
@@ -110,6 +90,7 @@ services:
op-batcher:
restart: always
image: cerc/optimism-op-batcher:local
hostname: op-batcher
depends_on:
op-node:
condition: service_healthy
@@ -129,7 +110,7 @@ services:
command: |
"/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-batcher.sh"
ports:
- "127.0.0.1:8548:8548"
- "8548"
extra_hosts:
- "host.docker.internal:host-gateway"
@@ -137,25 +118,29 @@ services:
op-proposer:
restart: always
image: cerc/optimism-op-proposer:local
hostname: op-proposer
depends_on:
op-node:
condition: service_healthy
op-geth:
condition: service_healthy
env_file:
- ../config/fixturenet-optimism/l1-params.env
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_L1_RPC: ${CERC_L1_RPC}
CERC_L1_CHAIN_ID: ${CERC_L1_CHAIN_ID}
volumes:
- ../config/network/wait-for-it.sh:/wait-for-it.sh
- ../config/fixturenet-optimism/run-op-proposer.sh:/run-op-proposer.sh
- l1_deployment:/contracts-bedrock:ro
- l1_deployment:/l1-deployment:ro
- l2_accounts:/l2-accounts:ro
entrypoint: ["sh", "-c"]
# Waits for L1 endpoint to be up before running the proposer
command: |
"/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-proposer.sh"
ports:
- "127.0.0.1:8560:8560"
- "8560"
extra_hosts:
- "host.docker.internal:host-gateway"
@@ -1,37 +0,0 @@
#!/bin/sh
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
# Check existing config if it exists
if [ -f /app/jwt.txt ] && [ -f /app/rollup.json ]; then
echo "Found existing L2 config, cross-checking with L1 deployment config"
SOURCE_L1_CONF=$(cat /contracts-bedrock/deploy-config/getting-started.json)
EXP_L1_BLOCKHASH=$(echo "$SOURCE_L1_CONF" | jq -r '.l1StartingBlockTag')
EXP_BATCHER=$(echo "$SOURCE_L1_CONF" | jq -r '.batchSenderAddress')
GEN_L2_CONF=$(cat /app/rollup.json)
GEN_L1_BLOCKHASH=$(echo "$GEN_L2_CONF" | jq -r '.genesis.l1.hash')
GEN_BATCHER=$(echo "$GEN_L2_CONF" | jq -r '.genesis.system_config.batcherAddr')
if [ "$EXP_L1_BLOCKHASH" = "$GEN_L1_BLOCKHASH" ] && [ "$EXP_BATCHER" = "$GEN_BATCHER" ]; then
echo "Config cross-checked, exiting"
exit 0
fi
echo "Existing L2 config doesn't match the L1 deployment config, please clear L2 config volume before starting"
exit 1
fi
op-node genesis l2 \
--deploy-config /contracts-bedrock/deploy-config/getting-started.json \
--deployment-dir /contracts-bedrock/deployments/getting-started/ \
--outfile.l2 /app/genesis.json \
--outfile.rollup /app/rollup.json \
--l1-rpc $CERC_L1_RPC
openssl rand -hex 32 > /app/jwt.txt
@@ -0,0 +1,172 @@
#!/bin/bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"
export DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID"
# Optional create2 salt for deterministic deployment of contract implementations
export IMPL_SALT=$(openssl rand -hex 32)
echo "Using L1 RPC endpoint ${CERC_L1_RPC}"
# Exit if a deployment already exists (on restarts)
if [ -d "/l1-deployment/$DEPLOYMENT_CONTEXT" ]; then
echo "Deployment directory /l1-deployment/$DEPLOYMENT_CONTEXT, checking OptimismPortal deployment"
OPTIMISM_PORTAL_ADDRESS=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/OptimismPortal.json | jq -r .address)
contract_code=$(cast code $OPTIMISM_PORTAL_ADDRESS --rpc-url $CERC_L1_RPC)
if [ -z "${contract_code#0x}" ]; then
echo "Error: A deployment directory was found in the volume, but no contract code was found on-chain at the associated address. Please clear L1 deployment volume before restarting."
exit 1
else
echo "Deployment found, exiting (successfully)."
exit 0
fi
fi
wait_for_block() {
local block="$1" # Block to wait for
local timeout="$2" # Max time to wait in seconds
echo "Waiting for block $block."
i=0
loops=$(($timeout/10))
while [ -z "$block_result" ] && [[ "$i" -lt "$loops" ]]; do
sleep 10
echo "Checking..."
block_result=$(cast block $block --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true)
i=$(($i + 1))
done
}
# We need four accounts and their private keys for the deployment: Admin, Proposer, Batcher, and Sequencer
# If $CERC_L1_ADDRESS and $CERC_L1_PRIV_KEY have been set, we'll assign it to Admin and generate/fund the remaining three accounts from it
# If not, we'll assume the L1 is the stack's own fixturenet-eth and use the pre-funded accounts/keys from $CERC_L1_ACCOUNTS_CSV_URL
if [ -n "$CERC_L1_ADDRESS" ] && [ -n "$CERC_L1_PRIV_KEY" ]; then
wallet1=$(cast wallet new)
wallet2=$(cast wallet new)
wallet3=$(cast wallet new)
# Admin
ADMIN=$CERC_L1_ADDRESS
ADMIN_KEY=$CERC_L1_PRIV_KEY
# Proposer
PROPOSER=$(echo "$wallet1" | awk '/Address:/{print $2}')
PROPOSER_KEY=$(echo "$wallet1" | awk '/Private key:/{print $3}')
# Batcher
BATCHER=$(echo "$wallet2" | awk '/Address:/{print $2}')
BATCHER_KEY=$(echo "$wallet2" | awk '/Private key:/{print $3}')
# Sequencer
SEQ=$(echo "$wallet3" | awk '/Address:/{print $2}')
SEQ_KEY=$(echo "$wallet3" | awk '/Private key:/{print $3}')
echo "Funding accounts."
wait_for_block 1 300
cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 5ether $PROPOSER --private-key $ADMIN_KEY
cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 10ether $BATCHER --private-key $ADMIN_KEY
cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 2ether $SEQ --private-key $ADMIN_KEY
else
curl -o accounts.csv $CERC_L1_ACCOUNTS_CSV_URL
# Admin
ADMIN=$(awk -F ',' 'NR == 1 {print $2}' accounts.csv)
ADMIN_KEY=$(awk -F ',' 'NR == 1 {print $3}' accounts.csv)
# Proposer
PROPOSER=$(awk -F ',' 'NR == 2 {print $2}' accounts.csv)
PROPOSER_KEY=$(awk -F ',' 'NR == 2 {print $3}' accounts.csv)
# Batcher
BATCHER=$(awk -F ',' 'NR == 3 {print $2}' accounts.csv)
BATCHER_KEY=$(awk -F ',' 'NR == 3 {print $3}' accounts.csv)
# Sequencer
SEQ=$(awk -F ',' 'NR == 4 {print $2}' accounts.csv)
SEQ_KEY=$(awk -F ',' 'NR == 4 {print $3}' accounts.csv)
fi
echo "Using accounts:"
echo -e "Admin: $ADMIN\nProposer: $PROPOSER\nBatcher: $BATCHER\nSequencer: $SEQ"
# These accounts will be needed by other containers, so write them to a shared volume
echo "Writing accounts/private keys to volume l2_accounts."
accounts_json=$(jq -n \
--arg Admin "$ADMIN" --arg AdminKey "$ADMIN_KEY" \
--arg Proposer "$PROPOSER" --arg ProposerKey "$PROPOSER_KEY" \
--arg Batcher "$BATCHER" --arg BatcherKey "$BATCHER_KEY" \
--arg Seq "$SEQ" --arg SeqKey "$SEQ_KEY" \
'{Admin: $Admin, AdminKey: $AdminKey, Proposer: $Proposer, ProposerKey: $ProposerKey, Batcher: $Batcher, BatcherKey: $BatcherKey, Seq: $Seq, SeqKey: $SeqKey}')
echo "$accounts_json" > "/l2-accounts/accounts.json"
# Get a finalized L1 block to set as the starting point for the L2 deployment
# If the chain is a freshly created fixturenet-eth, a finalized block won't be available for many minutes; rather than wait, we can use block 1
echo "Checking L1 for finalized block..."
finalized=$(cast block finalized --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true)
if [ -n "$finalized" ]; then
# finalized block was found
start_block=$finalized
else
# assume fresh chain and use block 1 instead
echo "No finalized block. Using block 1 instead."
# wait for 20 or so blocks to be safe
wait_for_block 24 300
start_block=$(cast block 1 --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true)
fi
if [ -z "$start_block" ]; then
echo "Unable to query chain for starting block. Exiting..."
exit 1
fi
BLOCKHASH=$(echo $start_block | awk -F ' ' '{print $2}')
HEIGHT=$(echo $start_block | awk -F ' ' '{print $4}')
TIMESTAMP=$(echo $start_block | awk -F ' ' '{print $6}')
echo "Using block as deployment point:"
echo "Height: $HEIGHT"
echo "Hash: $BLOCKHASH"
echo "Timestamp: $TIMESTAMP"
# Fill out the deployment template (./deploy-config/getting-started.json) with our values:
echo "Writing deployment config."
deploy_config_file="deploy-config/$DEPLOYMENT_CONTEXT.json"
cp deploy-config/getting-started.json $deploy_config_file
sed -i "s/\"l1ChainID\": .*/\"l1ChainID\": $DEPLOYMENT_CONTEXT,/g" $deploy_config_file
sed -i "s/ADMIN/$ADMIN/g" $deploy_config_file
sed -i "s/PROPOSER/$PROPOSER/g" $deploy_config_file
sed -i "s/BATCHER/$BATCHER/g" $deploy_config_file
sed -i "s/SEQUENCER/$SEQ/g" $deploy_config_file
sed -i "s/BLOCKHASH/$BLOCKHASH/g" $deploy_config_file
sed -i "s/TIMESTAMP/$TIMESTAMP/g" $deploy_config_file
mkdir -p deployments/$DEPLOYMENT_CONTEXT
# Deployment requires the create2 deterministic proxy contract be published on L1 at address 0x4e59b44847b379578588920ca78fbf26c0b4956c
# See: https://github.com/Arachnid/deterministic-deployment-proxy
echo "Deploying create2 proxy contract..."
echo "Funding deployment signer address"
deployment_signer="0x3fab184622dc19b6109349b94811493bf2a45362"
cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 0.5ether $deployment_signer --private-key $ADMIN_KEY
echo "Deploying contract..."
raw_bytes="0xf8a58085174876e800830186a08080b853604580600e600039806000f350fe7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf31ba02222222222222222222222222222222222222222222222222222222222222222a02222222222222222222222222222222222222222222222222222222222222222"
cast publish --rpc-url $CERC_L1_RPC $raw_bytes
# Create the L2 deployment
echo "Deploying L1 Optimism contracts..."
forge script scripts/Deploy.s.sol:Deploy --private-key $ADMIN_KEY --broadcast --rpc-url $CERC_L1_RPC
forge script scripts/Deploy.s.sol:Deploy --sig 'sync()' --private-key $ADMIN_KEY --broadcast --rpc-url $CERC_L1_RPC
echo "*************************************"
echo "Done deploying contracts."
# Copy files needed by other containers to the appropriate shared volumes
echo "Copying deployment artifacts volume l1_deployment and deploy-config to volume l2_config"
cp -a /app/packages/contracts-bedrock/deployments/$DEPLOYMENT_CONTEXT /l1-deployment
cp /app/packages/contracts-bedrock/deploy-config/$DEPLOYMENT_CONTEXT.json /l2-config
openssl rand -hex 32 > /l2-config/l2-jwt.txt
echo "Deployment successful. Exiting"
@@ -1,131 +0,0 @@
#!/bin/bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"
echo "Using L1 RPC endpoint ${CERC_L1_RPC}"
IMPORT_1="import './verify-contract-deployment'"
IMPORT_2="import './rekey-json'"
IMPORT_3="import './send-balance'"
# Append mounted tasks to tasks/index.ts file if not present
if ! grep -Fxq "$IMPORT_1" tasks/index.ts; then
echo "$IMPORT_1" >> tasks/index.ts
echo "$IMPORT_2" >> tasks/index.ts
echo "$IMPORT_3" >> tasks/index.ts
fi
# Update the chainId in the hardhat config
sed -i "/getting-started/ {n; s/.*chainId.*/ chainId: $CERC_L1_CHAIN_ID,/}" hardhat.config.ts
# Exit if a deployment already exists (on restarts)
# Note: fixturenet-eth-geth currently starts fresh on a restart
if [ -d "deployments/getting-started" ]; then
echo "Deployment directory deployments/getting-started found, checking SystemDictator deployment"
# Read JSON file into variable
SYSTEM_DICTATOR_DETAILS=$(cat deployments/getting-started/SystemDictator.json)
# Parse JSON into variables
SYSTEM_DICTATOR_ADDRESS=$(echo "$SYSTEM_DICTATOR_DETAILS" | jq -r '.address')
SYSTEM_DICTATOR_TXHASH=$(echo "$SYSTEM_DICTATOR_DETAILS" | jq -r '.transactionHash')
if yarn hardhat verify-contract-deployment --contract "${SYSTEM_DICTATOR_ADDRESS}" --transaction-hash "${SYSTEM_DICTATOR_TXHASH}"; then
echo "Deployment verfication successful, exiting"
exit 0
else
echo "Deployment verfication failed, please clear L1 deployment volume before starting"
exit 1
fi
fi
# Generate the L2 account addresses
yarn hardhat rekey-json --output /l2-accounts/keys.json
# Read JSON file into variable
KEYS_JSON=$(cat /l2-accounts/keys.json)
# Parse JSON into variables
ADMIN_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Admin.address')
ADMIN_PRIV_KEY=$(echo "$KEYS_JSON" | jq -r '.Admin.privateKey')
PROPOSER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Proposer.address')
BATCHER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Batcher.address')
SEQUENCER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Sequencer.address')
# Get the private keys of L1 accounts
if [ -n "$CERC_L1_ACCOUNTS_CSV_URL" ] && \
l1_accounts_response=$(curl -L --write-out '%{http_code}' --silent --output /dev/null "$CERC_L1_ACCOUNTS_CSV_URL") && \
[ "$l1_accounts_response" -eq 200 ];
then
echo "Fetching L1 account credentials using provided URL"
mkdir -p /geth-accounts
wget -O /geth-accounts/accounts.csv "$CERC_L1_ACCOUNTS_CSV_URL"
CERC_L1_ADDRESS=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 2)
CERC_L1_PRIV_KEY=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 3)
CERC_L1_ADDRESS_2=$(awk -F, 'NR==2{print $(NF-1)}' /geth-accounts/accounts.csv)
CERC_L1_PRIV_KEY_2=$(awk -F, 'NR==2{print $NF}' /geth-accounts/accounts.csv)
else
echo "Couldn't fetch L1 account credentials, using them from env"
fi
# Send balances to the above L2 addresses
yarn hardhat send-balance --to "${ADMIN_ADDRESS}" --amount 2 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started
yarn hardhat send-balance --to "${PROPOSER_ADDRESS}" --amount 5 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started
yarn hardhat send-balance --to "${BATCHER_ADDRESS}" --amount 1000 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started
echo "Balances sent to L2 accounts"
# Select a finalized L1 block as the starting point for roll ups
until FINALIZED_BLOCK=$(cast block finalized --rpc-url "$CERC_L1_RPC"); do
echo "Waiting for a finalized L1 block to exist, retrying after 10s"
sleep 10
done
L1_BLOCKNUMBER=$(echo "$FINALIZED_BLOCK" | awk '/number/{print $2}')
L1_BLOCKHASH=$(echo "$FINALIZED_BLOCK" | awk '/hash/{print $2}')
L1_BLOCKTIMESTAMP=$(echo "$FINALIZED_BLOCK" | awk '/timestamp/{print $2}')
echo "Selected L1 block ${L1_BLOCKNUMBER} as the starting block for roll ups"
# Update the deployment config
sed -i 's/"l2OutputOracleStartingTimestamp": TIMESTAMP/"l2OutputOracleStartingTimestamp": '"$L1_BLOCKTIMESTAMP"'/g' deploy-config/getting-started.json
jq --arg chainid "$CERC_L1_CHAIN_ID" '.l1ChainID = ($chainid | tonumber)' deploy-config/getting-started.json > tmp.json && mv tmp.json deploy-config/getting-started.json
node update-config.js deploy-config/getting-started.json "$ADMIN_ADDRESS" "$PROPOSER_ADDRESS" "$BATCHER_ADDRESS" "$SEQUENCER_ADDRESS" "$L1_BLOCKHASH"
echo "Updated the deployment config"
# Create a .env file
echo "L1_RPC=$CERC_L1_RPC" > .env
echo "PRIVATE_KEY_DEPLOYER=$ADMIN_PRIV_KEY" >> .env
echo "Deploying the L1 smart contracts, this will take a while..."
# Deploy the L1 smart contracts
yarn hardhat deploy --network getting-started --tags l1
echo "Deployed the L1 smart contracts"
# Read Proxy contract's JSON and get the address
PROXY_JSON=$(cat deployments/getting-started/Proxy__OVM_L1StandardBridge.json)
PROXY_ADDRESS=$(echo "$PROXY_JSON" | jq -r '.address')
# Send balance to the above Proxy contract in L1 for reflecting balance in L2
# First account
yarn hardhat send-balance --to "${PROXY_ADDRESS}" --amount 1 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started
# Second account
yarn hardhat send-balance --to "${PROXY_ADDRESS}" --amount 1 --private-key "${CERC_L1_PRIV_KEY_2}" --network getting-started
echo "Balance sent to Proxy L2 contract"
echo "Use following accounts for transactions in L2:"
echo "${CERC_L1_ADDRESS}"
echo "${CERC_L1_ADDRESS_2}"
echo "Done"
@@ -1,36 +0,0 @@
const fs = require('fs')
// Get the command-line argument
const configFile = process.argv[2]
const adminAddress = process.argv[3]
const proposerAddress = process.argv[4]
const batcherAddress = process.argv[5]
const sequencerAddress = process.argv[6]
const blockHash = process.argv[7]
// Read the JSON file
const configData = fs.readFileSync(configFile)
const configObj = JSON.parse(configData)
// Update the finalSystemOwner property with the ADMIN_ADDRESS value
configObj.finalSystemOwner =
configObj.portalGuardian =
configObj.controller =
configObj.l2OutputOracleChallenger =
configObj.proxyAdminOwner =
configObj.baseFeeVaultRecipient =
configObj.l1FeeVaultRecipient =
configObj.sequencerFeeVaultRecipient =
configObj.governanceTokenOwner =
adminAddress
configObj.l2OutputOracleProposer = proposerAddress
configObj.batchSenderAddress = batcherAddress
configObj.p2pSequencerAddress = sequencerAddress
configObj.l1StartingBlockTag = blockHash
// Write the updated JSON object back to the file
fs.writeFileSync(configFile, JSON.stringify(configObj, null, 2))
@@ -0,0 +1,155 @@
#!/bin/bash
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
# To facilitate deploying the Optimism contracts, a few additional arguments have been added to the geth start command
# Otherwise this script is unchanged from the image's default startup script
ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}"
CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}"
cd /opt/testnet/build/el
python3 -m http.server 9898 &
cd $HOME
START_CMD="geth"
if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --"
fi
# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script
cleanup() {
echo "Signal received, cleaning up..."
# Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process)
pkill -P ${geth_pid}
sleep 2
kill $(jobs -p)
wait
echo "Done"
}
trap 'cleanup' SIGINT SIGTERM
if [ "true" == "$RUN_BOOTNODE" ]; then
$START_CMD \
--datadir="${CERC_ETH_DATADIR}" \
--nodekeyhex="${BOOTNODE_KEY}" \
--nodiscover \
--ipcdisable \
--networkid=${NETWORK_ID} \
--netrestrict="${NETRESTRICT}" \
&
geth_pid=$!
else
cd /opt/testnet/accounts
./import_keys.sh
echo -n "$JWT" > /opt/testnet/build/el/jwtsecret
if [ "$CERC_RUN_STATEDIFF" == "detect" ] && [ -n "$CERC_STATEDIFF_DB_HOST" ]; then
dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short)
dig_status_code=$?
if [[ $dig_status_code = 0 && -n $dig_result ]]; then
echo "Statediff DB at $CERC_STATEDIFF_DB_HOST"
CERC_RUN_STATEDIFF="true"
else
echo "No statediff DB available."
CERC_RUN_STATEDIFF="false"
fi
fi
STATEDIFF_OPTS=""
if [ "$CERC_RUN_STATEDIFF" == "true" ]; then
ready=0
echo "Waiting for statediff DB..."
while [ $ready -eq 0 ]; do
sleep 1
export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD"
result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \
-p "$CERC_STATEDIFF_DB_PORT" \
-U "$CERC_STATEDIFF_DB_USER" \
-d "$CERC_STATEDIFF_DB_NAME" \
-t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }')
if [ -n "$result" ]; then
echo "DB ready..."
if [ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]; then
ready=1
else
echo "DB not at required version (want $CERC_STATEDIFF_DB_GOOSE_MIN_VER, have $result)"
fi
fi
done
STATEDIFF_OPTS="--statediff \
--statediff.db.host=$CERC_STATEDIFF_DB_HOST \
--statediff.db.name=$CERC_STATEDIFF_DB_NAME \
--statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \
--statediff.db.password=$CERC_STATEDIFF_DB_PASSWORD \
--statediff.db.port=$CERC_STATEDIFF_DB_PORT \
--statediff.db.user=$CERC_STATEDIFF_DB_USER \
--statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \
--statediff.db.copyfrom=${CERC_STATEDIFF_DB_COPY_FROM:-true} \
--statediff.waitforsync=true \
--statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \
--statediff.writing=true"
if [ -d "${CERC_PLUGINS_DIR}" ]; then
# With plugeth, we separate the statediff options by prefixing with ' -- '
STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}"
fi
fi
# unlock account[0]
echo $ACCOUNT_PASSWORD > "$CERC_ETH_DATADIR/password"
$START_CMD \
--datadir="${CERC_ETH_DATADIR}" \
--bootnodes="${ENODE}" \
--allow-insecure-unlock \
--password="${CERC_ETH_DATADIR}/password" \
--unlock="$ETHERBASE" \
--rpc.allow-unprotected-txs \
--http \
--http.addr="0.0.0.0" \
--http.vhosts="*" \
--http.api="${CERC_GETH_HTTP_APIS:-eth,web3,net,admin,personal,debug,statediff}" \
--http.corsdomain="*" \
--authrpc.addr="0.0.0.0" \
--authrpc.vhosts="*" \
--authrpc.jwtsecret="/opt/testnet/build/el/jwtsecret" \
--ws \
--ws.addr="0.0.0.0" \
--ws.origins="*" \
--ws.api="${CERC_GETH_WS_APIS:-eth,web3,net,admin,personal,debug,statediff}" \
--http.corsdomain="*" \
--networkid="${NETWORK_ID}" \
--netrestrict="${NETRESTRICT}" \
--gcmode archive \
--txlookuplimit=0 \
--cache.preimages \
--syncmode=full \
--mine \
--miner.threads=1 \
--metrics \
--metrics.addr="0.0.0.0" \
--verbosity=${CERC_GETH_VERBOSITY:-3} \
--log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
--miner.etherbase="${ETHERBASE}" \
${STATEDIFF_OPTS} \
&
geth_pid=$!
fi
wait $geth_pid
if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then
while [ 1 -eq 1 ]; do
sleep 60
done
fi
@@ -6,22 +6,14 @@ fi
CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
# Get Batcher key from keys.json
BATCHER_KEY=$(jq -r '.Batcher.privateKey' /l2-accounts/keys.json | tr -d '"')
# Start op-batcher
L2_RPC="http://op-geth:8545"
ROLLUP_RPC="http://op-node:8547"
BATCHER_KEY=$(cat /l2-accounts/accounts.json | jq -r .BatcherKey)
cleanup() {
echo "Signal received, cleaning up..."
kill ${batcher_pid}
wait
echo "Done"
}
trap 'cleanup' INT TERM
# Run op-batcher
op-batcher \
--l2-eth-rpc=http://op-geth:8545 \
--rollup-rpc=http://op-node:8547 \
--l2-eth-rpc=$L2_RPC \
--rollup-rpc=$ROLLUP_RPC \
--poll-interval=1s \
--sub-safety-margin=6 \
--num-confirmations=1 \
@@ -32,8 +24,4 @@ op-batcher \
--rpc.enable-admin \
--max-channel-duration=1 \
--l1-eth-rpc=$CERC_L1_RPC \
--private-key=$BATCHER_KEY \
&
batcher_pid=$!
wait $batcher_pid
--private-key="${BATCHER_KEY#0x}"
@@ -4,61 +4,36 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
# TODO: Add in container build or use other tool
echo "Installing jq"
apk update && apk add jq
l2_genesis_file="/l2-config/genesis.json"
# Get Sequencer key from keys.json
SEQUENCER_KEY=$(jq -r '.Sequencer.privateKey' /l2-accounts/keys.json | tr -d '"')
# Check for genesis file; if necessary, wait on op-node to generate
timeout=300 # 5 minutes
start_time=$(date +%s)
elapsed_time=0
echo "Checking for L2 genesis file at location $l2_genesis_file"
while [ ! -f "$l2_genesis_file" ] && [ $elapsed_time -lt $timeout ]; do
echo "Waiting for L2 genesis file to be generated..."
sleep 10
current_time=$(date +%s)
elapsed_time=$((current_time - start_time))
done
# Initialize op-geth if datadir/geth not found
if [ -f /op-node/jwt.txt ] && [ -d datadir/geth ]; then
echo "Found existing datadir, checking block signer key"
BLOCK_SIGNER_KEY=$(cat datadir/block-signer-key)
if [ "$SEQUENCER_KEY" = "$BLOCK_SIGNER_KEY" ]; then
echo "Sequencer and block signer keys match, skipping initialization"
else
echo "Sequencer and block signer keys don't match, please clear L2 geth data volume before starting"
exit 1
fi
else
echo "Initializing op-geth"
mkdir -p datadir
echo "pwd" > datadir/password
echo $SEQUENCER_KEY > datadir/block-signer-key
geth account import --datadir=datadir --password=datadir/password datadir/block-signer-key
while [ ! -f "/op-node/jwt.txt" ]
do
echo "Config files not created. Checking after 5 seconds."
sleep 5
done
echo "Config files created by op-node, proceeding with the initialization..."
geth init --datadir=datadir /op-node/genesis.json
echo "Node Initialized"
if [ ! -f "$l2_genesis_file" ]; then
echo "L2 genesis file not found after timeout of $timeout seconds. Exiting..."
exit 1
fi
SEQUENCER_ADDRESS=$(jq -r '.Sequencer.address' /l2-accounts/keys.json | tr -d '"')
echo "SEQUENCER_ADDRESS: ${SEQUENCER_ADDRESS}"
# Initialize geth from our generated L2 genesis file (if not already initialized)
data_dir="/datadir"
if [ ! -d "$datadir/geth" ]; then
geth init --datadir=$data_dir $l2_genesis_file
fi
cleanup() {
echo "Signal received, cleaning up..."
kill ${geth_pid}
# Start op-geth
jwt_file="/l2-config/l2-jwt.txt"
wait
echo "Done"
}
trap 'cleanup' INT TERM
# Run op-geth
geth \
--datadir ./datadir \
--datadir=$data_dir \
--http \
--http.corsdomain="*" \
--http.vhosts="*" \
@@ -77,14 +52,5 @@ geth \
--authrpc.vhosts="*" \
--authrpc.addr=0.0.0.0 \
--authrpc.port=8551 \
--authrpc.jwtsecret=/op-node/jwt.txt \
--rollup.disabletxpoolgossip=true \
--password=./datadir/password \
--allow-insecure-unlock \
--mine \
--miner.etherbase=$SEQUENCER_ADDRESS \
--unlock=$SEQUENCER_ADDRESS \
&
geth_pid=$!
wait $geth_pid
--authrpc.jwtsecret=$jwt_file \
--rollup.disabletxpoolgossip=true
@@ -4,23 +4,42 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID"
# Get Sequencer key from keys.json
SEQUENCER_KEY=$(jq -r '.Sequencer.privateKey' /l2-accounts/keys.json | tr -d '"')
deploy_config_file="/l2-config/$DEPLOYMENT_CONTEXT.json"
deployment_dir="/l1-deployment/$DEPLOYMENT_CONTEXT"
genesis_outfile="/l2-config/genesis.json"
rollup_outfile="/l2-config/rollup.json"
# Generate L2 genesis (if not already done)
if [ ! -f "$genesis_outfile" ] || [ ! -f "$rollup_outfile" ]; then
op-node genesis l2 \
--deploy-config $deploy_config_file \
--deployment-dir $deployment_dir \
--outfile.l2 $genesis_outfile \
--outfile.rollup $rollup_outfile \
--l1-rpc $CERC_L1_RPC
fi
# Start op-node
SEQ_KEY=$(cat /l2-accounts/accounts.json | jq -r .SeqKey)
jwt_file=/l2-config/l2-jwt.txt
L2_AUTH="http://op-geth:8551"
RPC_KIND=any # this can optionally be set to a preset for common node providers like Infura, Alchemy, etc.
# Run op-node
op-node \
--l2=http://op-geth:8551 \
--l2.jwt-secret=/op-node-data/jwt.txt \
--sequencer.enabled \
--sequencer.l1-confs=3 \
--verifier.l1-confs=3 \
--rollup.config=/op-node-data/rollup.json \
--rpc.addr=0.0.0.0 \
--rpc.port=8547 \
--p2p.disable \
--rpc.enable-admin \
--p2p.sequencer.key=$SEQUENCER_KEY \
--l1=$CERC_L1_RPC \
--l1.rpckind=any
--l2=$L2_AUTH \
--l2.jwt-secret=$jwt_file \
--sequencer.enabled \
--sequencer.l1-confs=5 \
--verifier.l1-confs=4 \
--rollup.config=$rollup_outfile \
--rpc.addr=0.0.0.0 \
--rpc.port=8547 \
--p2p.disable \
--rpc.enable-admin \
--p2p.sequencer.key="${SEQ_KEY#0x}" \
--l1=$CERC_L1_RPC \
--l1.rpckind=$RPC_KIND
@@ -5,32 +5,18 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID"
# Read the L2OutputOracle contract address from the deployment
L2OO_DEPLOYMENT=$(cat /contracts-bedrock/deployments/getting-started/L2OutputOracle.json)
L2OO_ADDR=$(echo "$L2OO_DEPLOYMENT" | jq -r '.address')
# Start op-proposer
ROLLUP_RPC="http://op-node:8547"
PROPOSER_KEY=$(cat /l2-accounts/accounts.json | jq -r .ProposerKey)
L2OO_ADDR=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/L2OutputOracleProxy.json | jq -r .address)
# Get Proposer key from keys.json
PROPOSER_KEY=$(jq -r '.Proposer.privateKey' /l2-accounts/keys.json | tr -d '"')
cleanup() {
echo "Signal received, cleaning up..."
kill ${proposer_pid}
wait
echo "Done"
}
trap 'cleanup' INT TERM
# Run op-proposer
op-proposer \
--poll-interval 12s \
--rpc.port 8560 \
--rollup-rpc http://op-node:8547 \
--l2oo-address $L2OO_ADDR \
--private-key $PROPOSER_KEY \
--l1-eth-rpc $CERC_L1_RPC \
&
proposer_pid=$!
wait $proposer_pid
--poll-interval=12s \
--rpc.port=8560 \
--rollup-rpc=$ROLLUP_RPC \
--l2oo-address="${L2OO_ADDR#0x}" \
--private-key="${PROPOSER_KEY#0x}" \
--l1-eth-rpc=$CERC_L1_RPC
@@ -0,0 +1,55 @@
# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
ARG VARIANT=20-bullseye
FROM node:${VARIANT}
ARG USERNAME=node
ARG NPM_GLOBAL=/usr/local/share/npm-global
# Add NPM global to PATH.
ENV PATH=${NPM_GLOBAL}/bin:${PATH}
# Prevents npm from printing version warnings
ENV NPM_CONFIG_UPDATE_NOTIFIER=false
RUN \
# Configure global npm install location, use group to adapt to UID/GID changes
if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \
&& usermod -a -G npm ${USERNAME} \
&& umask 0002 \
&& mkdir -p ${NPM_GLOBAL} \
&& touch /usr/local/etc/npmrc \
&& chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \
&& chmod g+s ${NPM_GLOBAL} \
&& npm config -g set prefix ${NPM_GLOBAL} \
&& su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
# Install eslint
&& su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
&& npm cache clean --force > /dev/null 2>&1
# [Optional] Uncomment this section to install additional OS packages.
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
&& apt-get -y install --no-install-recommends jq gettext-base
# [Optional] Uncomment if you want to install an additional version of node using nvm
# ARG EXTRA_NODE_VERSION=10
# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
# We do this to get a yq binary from the published container, for the correct architecture we're building here
# COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
COPY /scripts /scripts
# [Optional] Uncomment if you want to install more global node modules
# RUN su node -c "npm install -g <your-package-list-here>"
# RUN mkdir -p /config
# COPY ./config.yml /config
# Install simple web server for now (use nginx perhaps later)
# RUN yarn global add http-server
# Expose port for http
EXPOSE 3000
# Default command sleeps forever so docker doesn't kill it
CMD ["/scripts/start-serving-app.sh"]
@@ -0,0 +1,5 @@
FROM cerc/nextjs-base:local
WORKDIR /app
COPY . .
RUN rm -rf node_modules build .next*
RUN /scripts/build-app.sh /app
@@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Build cerc/laconic-registry-cli
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
# See: https://stackoverflow.com/a/246128/1701505
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
CERC_CONTAINER_BUILD_WORK_DIR=${CERC_CONTAINER_BUILD_WORK_DIR:-$SCRIPT_DIR}
CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/Dockerfile}
CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}
docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
@@ -0,0 +1,36 @@
#!/bin/bash
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
WORK_DIR="${1:-./}"
SRC_DIR="${2:-.next}"
TRG_DIR="${3:-.next-r}"
cd "${WORK_DIR}" || exit 1
rm -rf "$TRG_DIR"
mkdir -p "$TRG_DIR"
cp -rp "$SRC_DIR" "$TRG_DIR/"
if [ -f ".env" ]; then
TMP_ENV=`mktemp`
declare -px > $TMP_ENV
set -a
source .env
source $TMP_ENV
set +a
rm -f $TMP_ENV
fi
for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o '^"CERC_RUNTIME_ENV_[^\"]+"'); do
orig_name=$(echo -n "${e}" | sed 's/"//g')
cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
cur_val=$(echo -n "\$${cur_name}" | envsubst)
esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
echo "$f: $cur_name=$cur_val"
sed -i "s/$orig_name/$esc_val/g" $f
done
done
@@ -0,0 +1,94 @@
#!/bin/bash
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
WORK_DIR="${1:-/app}"
cd "${WORK_DIR}" || exit 1
if [ ! -f "next.config.dist" ]; then
cp next.config.js next.config.dist
fi
which js-beautify >/dev/null
if [ $? -ne 0 ]; then
npm i -g js-beautify
fi
js-beautify next.config.dist > next.config.js
WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" next.config.js | cut -d':' -f1)
if [ -z "$WEBPACK_REQ_LINE" ]; then
cat > next.config.js.0 <<EOF
const webpack = require('webpack');
EOF
fi
cat > next.config.js.1 <<EOF
let envMap;
try {
// .env-list.json provides us a list of identifiers which should be replaced at runtime.
envMap = require('./.env-list.json').reduce((a, v) => {
a[v] = \`"CERC_RUNTIME_ENV_\${v.split(/\./).pop()}"\`;
return a;
}, {});
} catch {
// If .env-list.json cannot be loaded, we are probably running in dev mode, so use process.env instead.
envMap = Object.keys(process.env).reduce((a, v) => {
if (v.startsWith('CERC_')) {
a[\`process.env.\${v}\`] = JSON.stringify(process.env[v]);
}
return a;
}, {});
}
EOF
CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }')
ENV_LINE=$(grep -n 'env:' next.config.js | cut -d':' -f1)
WEBPACK_CONF_LINE=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d':' -f1)
NEXT_SECTION_ADJUSTMENT=0
if [ -n "$WEBPACK_CONF_LINE" ]; then
WEBPACK_CONF_VAR=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d',' -f1 | cut -d'(' -f2)
head -$(( ${WEBPACK_CONF_LINE} )) next.config.js > next.config.js.2
cat > next.config.js.3 <<EOF
$WEBPACK_CONF_VAR.plugins.push(new webpack.DefinePlugin(envMap));
EOF
NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE - 1))
elif [ -n "$ENV_LINE" ]; then
head -$(( ${ENV_LINE} - 1 )) next.config.js > next.config.js.2
cat > next.config.js.3 <<EOF
webpack: (config) => {
config.plugins.push(new webpack.DefinePlugin(envMap));
return config;
},
EOF
NEXT_SECTION_ADJUSTMENT=2
NEXT_SECTION_LINE=$ENV_LINE
else
echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2
rm -f next.config.js.*
NEXT_SECTION_LINE=0
fi
tail -$(( ${CONFIG_LINES} - ${NEXT_SECTION_LINE} + ${NEXT_SECTION_ADJUSTMENT} )) next.config.js > next.config.js.5
cat next.config.js.* | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify > next.config.js
rm next.config.js.*
"${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json
if [ ! -f "package.dist" ]; then
cp package.json package.dist
fi
cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json
npm install || exit 1
npm run cerc_compile || exit 1
exit 0
@@ -0,0 +1,29 @@
#!/bin/bash
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
WORK_DIR="${1:-./}"
TMPF=$(mktemp)
cd "$WORK_DIR" || exit 1
for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' -f2); do
egrep "/$d[/$]?" .gitignore >/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
continue
fi
for f in $(find "$d" -regex ".*.[tj]sx?$" -type f); do
cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF
done
done
NEXT_CONF="next.config.js next.config.dist"
for f in $NEXT_CONF; do
cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF
done
cat $TMPF | sort -u | jq --raw-input . | jq --slurp .
rm -f $TMPF
@@ -0,0 +1,34 @@
#!/usr/bin/env bash
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}"
cd "$CERC_WEBAPP_FILES_DIR"
"$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r
mv .next .next.old
mv .next-r/.next .
if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
jq -e '.scripts.cerc_generate' package.json >/dev/null
if [ $? -eq 0 ]; then
npm run cerc_generate > gen.out 2>&1 &
tail -n0 -f gen.out | sed '/rendered as static HTML/ q'
count=0
while [ $count -lt 10 ]; do
sleep 1
ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null
if [ $? -ne 0 ]; then
break
else
count=$((count + 1))
fi
done
kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
fi
fi
npm start . -p ${CERC_LISTEN_PORT:-3000}
@@ -17,6 +17,6 @@ WORKDIR /app
COPY . .
RUN echo "Building optimism" && \
yarn && yarn build
pnpm install && pnpm build
WORKDIR /app/packages/contracts-bedrock
@@ -1,4 +1,4 @@
FROM golang:1.19.0-alpine3.15 as builder
FROM golang:1.21.0-alpine3.18 as builder
ARG VERSION=v0.0.0
@@ -9,7 +9,7 @@ COPY ./op-batcher /app/op-batcher
COPY ./op-bindings /app/op-bindings
COPY ./op-node /app/op-node
COPY ./op-service /app/op-service
COPY ./op-signer /app/op-signer
#COPY ./op-signer /app/op-signer
COPY ./go.mod /app/go.mod
COPY ./go.sum /app/go.sum
@@ -23,7 +23,7 @@ ARG TARGETOS TARGETARCH
RUN make op-batcher VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH
FROM alpine:3.15
FROM alpine:3.18
RUN apk add --no-cache jq bash
@@ -1,4 +1,4 @@
FROM golang:1.19.0-alpine3.15 as builder
FROM golang:1.21.0-alpine3.18 as builder
ARG VERSION=v0.0.0
@@ -21,7 +21,7 @@ ARG TARGETOS TARGETARCH
RUN make op-node VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH
FROM alpine:3.15
FROM alpine:3.18
RUN apk add --no-cache openssl jq
@@ -1,4 +1,4 @@
FROM golang:1.19.0-alpine3.15 as builder
FROM golang:1.21.0-alpine3.18 as builder
ARG VERSION=v0.0.0
@@ -9,7 +9,7 @@ COPY ./op-proposer /app/op-proposer
COPY ./op-bindings /app/op-bindings
COPY ./op-node /app/op-node
COPY ./op-service /app/op-service
COPY ./op-signer /app/op-signer
#COPY ./op-signer /app/op-signer
COPY ./go.mod /app/go.mod
COPY ./go.sum /app/go.sum
COPY ./.git /app/.git
@@ -22,7 +22,7 @@ ARG TARGETOS TARGETARCH
RUN make op-proposer VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH
FROM alpine:3.15
FROM alpine:3.18
RUN apk add --no-cache jq bash
@@ -12,6 +12,7 @@ Clone required repositories:
laconic-so --stack fixturenet-optimism setup-repositories
# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
# The repositories are located in $HOME/cerc by default
```
Build the container images:
@@ -39,20 +40,53 @@ This should create the required docker images in the local image registry:
* `cerc/optimism-op-batcher`
* `cerc/optimism-op-proposer`
## Deploy
Deploy the stack:
## Create a deployment
First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
```bash
laconic-so --stack fixturenet-optimism deploy up
laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed-random --output fixturenet-optimism-spec.yml
```
The `fixturenet-optimism-contracts` service takes a while to complete running as it:
1. waits for the 'Merge' to happen on L1
2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups)
3. deploys the L1 contracts
It may restart a few times after running into errors.
### Ports
It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
In addition, a stack-wide port mapping "recipe" can be applied at the time the
`laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
| Recipe | Host Port Mapping |
|--------|-------------------|
| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `fixturenet-eth-geth-1` RPC to port 8545 and the `op-geth` RPC to port 9545 on the host.
Or, you may wish to use `any-same` for the initial mappings -- in which case you'll have to edit the spec to file to ensure the various geth instances aren't all trying to publish to host ports 8545/8546 at once.
### Data volumes
Container data volumes are bind-mounted to specified paths in the host filesystem.
The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
---
Once you've made any needed changes to the spec file, create a deployment from it:
```bash
laconic-so --stack fixturenet-optimism deploy create --spec-file fixturenet-optimism-spec.yml --deployment-dir fixturenet-optimism-deployment
```
## Start the stack
Start the deployment:
```bash
laconic-so deployment --dir fixturenet-optimism-deployment start
```
1. The `fixturenet-eth` L1 chain will start up first and begin producing blocks.
2. The `fixturenet-optimism-contracts` service will configure and deploy the Optimism contracts to L1, exiting when complete. This may take several minutes; you can follow the progress by following the container's logs (see below).
3. The `op-node` and `op-geth` services will initialize themselves (if not already initialized) and start
4. The remaining services, `op-batcher` and `op-proposer` will start
### Logs
To list and monitor the running containers:
```bash
@@ -65,22 +99,69 @@ docker ps
docker logs -f <CONTAINER_ID>
```
## Clean up
## Example: bridge some ETH from L1 to L2
Stop all services running in the background:
Send some ETH from the desired account to the `L1StandardBridgeProxy` contract on L1 to test bridging to L2.
We can use the testing account `0xe6CE22afe802CAf5fF7d3845cec8c736ecc8d61F` which is pre-funded and unlocked, and the `cerc/foundry:local` container to make use of the `cast` cli.
1. Note the docker network the stack is running on:
```bash
laconic-so --stack fixturenet-optimism deploy down 30
docker network ls
# The network name will be something like laconic-[some_hash]_default
```
2. Set some variables:
```bash
L1_RPC=http://fixturenet-eth-geth-1:8545
L2_RPC=http://op-geth:8545
NETWORK=<the network name found above>
DEPLOYMENT_CONTEXT=<L1 chain-id; 1212 by default>
ACCOUNT=0xe6CE22afe802CAf5fF7d3845cec8c736ecc8d61F
```
Clear volumes created by this stack:
If you need to check the L1 chain-id, you can use:
```bash
docker run --rm --network $NETWORK cerc/foundry:local "cast chain-id --rpc-url $L1_RPC"
```
3. Check the account starting balance on L2 (it should be 0):
```bash
docker run --rm --network $NETWORK cerc/foundry:local "cast balance $ACCOUNT --rpc-url $L2_RPC"
# 0
```
4. Read the bridge contract address from the L1 deployment records in the `op-node` container:
```bash
# get the container id for op-node
NODE_CONTAINER=$(docker ps --filter "name=op-node" -q)
BRIDGE=$(docker exec $NODE_CONTAINER cat /l1-deployment/$DEPLOYMENT_CONTEXT/L1StandardBridgeProxy.json | jq -r .address)
```
5. Use cast to send some ETH to the bridge contract:
```bash
docker run --rm --network $NETWORK cerc/foundry:local "cast send --from $ACCOUNT --value 1ether $BRIDGE --rpc-url $L1_RPC"
```
6. Allow a couple minutes for the bridge to complete
7. Check the L2 balance again (it should show the bridged funds):
```bash
docker run --rm --network $NETWORK cerc/foundry:local "cast balance $ACCOUNT --rpc-url $L2_RPC"
# 1000000000000000000
```
## Clean up
To stop all services running in the background, while preserving chain data:
```bash
# List all relevant volumes
docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"
laconic-so deployment --dir fixturenet-optimism-deployment stop
```
# Remove all the listed volumes
docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
To stop all services and also delete chain data:
```bash
laconic-so deployment --dir fixturenet-optimism-deployment stop --delete-volumes
```
## Troubleshooting
@@ -0,0 +1,39 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from stack_orchestrator.deploy.deployment_context import DeploymentContext
from ruamel.yaml import YAML
def create(context: DeploymentContext, extra_args):
# Slightly modify the base fixturenet-eth compose file to replace the startup script for fixturenet-eth-geth-1
# We need to start geth with the flag to allow non eip-155 compliant transactions in order to publish the
# deterministic-deployment-proxy contract, which itself is a prereq for Optimism contract deployment
fixturenet_eth_compose_file = context.deployment_dir.joinpath('compose', 'docker-compose-fixturenet-eth.yml')
with open(fixturenet_eth_compose_file, 'r') as yaml_file:
yaml = YAML()
yaml_data = yaml.load(yaml_file)
new_script = '../config/fixturenet-optimism/run-geth.sh:/opt/testnet/run.sh'
if new_script not in yaml_data['services']['fixturenet-eth-geth-1']['volumes']:
yaml_data['services']['fixturenet-eth-geth-1']['volumes'].append(new_script)
with open(fixturenet_eth_compose_file, 'w') as yaml_file:
yaml = YAML()
yaml.dump(yaml_data, yaml_file)
return None
@@ -1,4 +1,4 @@
# fixturenet-optimism
# fixturenet-optimism (L2-only)
Instructions to setup and deploy L2 fixturenet using [Optimism](https://stack.optimism.io)
@@ -28,9 +28,43 @@ This should create the required docker images in the local image registry:
* `cerc/optimism-op-batcher`
* `cerc/optimism-op-proposer`
## Deploy
## Create a deployment
Create and update an env file to be used in the next step ([defaults](../../config/fixturenet-optimism/l1-params.env)):
First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
```bash
laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed-random --output fixturenet-optimism-spec.yml
```
### Ports
It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
In addition, a stack-wide port mapping "recipe" can be applied at the time the
`laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
| Recipe | Host Port Mapping |
|--------|-------------------|
| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `op-geth` RPC to an easy to remember port like 8545 or 9545 on the host.
### Data volumes
Container data volumes are bind-mounted to specified paths in the host filesystem.
The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
---
Once you've made any needed changes to the spec file, create a deployment from it:
```bash
laconic-so --stack fixturenet-optimism deploy create --spec-file fixturenet-optimism-spec.yml --deployment-dir fixturenet-optimism-deployment
```
Finally, open the `stack.yml` file inside your deployment directory and, under the `pods:` section, remove (or comment out) the entry for `fixturenet-eth`. This will prevent the deployment from trying to spin up a new L1 chain when starting the stack.
## Set chain env variables
Inside the deployment directory, open the file `config.env` and add the following variables to point the stack at your L1 rpc and provide account credentials ([defaults](../../config/fixturenet-optimism/l1-params.env)):
```bash
# External L1 endpoint
@@ -45,30 +79,29 @@ Create and update an env file to be used in the next step ([defaults](../../conf
CERC_L1_ACCOUNTS_CSV_URL=
# OR
# Specify the required account credentials
# Specify the required account credentials for the Admin account
# Other generated accounts will be funded from this account, so it should contain ~20 Eth
CERC_L1_ADDRESS=
CERC_L1_PRIV_KEY=
CERC_L1_ADDRESS_2=
CERC_L1_PRIV_KEY_2=
```
* NOTE: If L1 is running on the host machine, use `host.docker.internal` as the hostname to access the host port
Deploy the stack:
* NOTE: If L1 is running on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`)
## Start the stack
Start the deployment:
```bash
laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism --env-file <PATH_TO_ENV_FILE> up
laconic-so deployment --dir fixturenet-optimism-deployment start
```
1. The stack will check for a response from the L1 endpoint specified in your env file.
2. The `fixturenet-optimism-contracts` service will configure and deploy the Optimism contracts to L1, exiting when complete. This may take several minutes; you can follow the progress by following the container's logs (see below).
3. The `op-node` and `op-geth` services will initialize themselves (if not already initialized) and start
4. The remaining services, `op-batcher` and `op-proposer` will start
The `fixturenet-optimism-contracts` service may take a while (`~15 mins`) to complete running as it:
1. waits for the 'Merge' to happen on L1
2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups)
3. deploys the L1 contracts
To list down and monitor the running containers:
### Logs
To list and monitor the running containers:
```bash
laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism ps
laconic-so --stack fixturenet-optimism deploy ps
# With status
docker ps
@@ -79,20 +112,16 @@ docker logs -f <CONTAINER_ID>
## Clean up
Stop all services running in the background:
To stop all L2 services running in the background, while preserving chain data:
```bash
laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism down 30
laconic-so deployment --dir fixturenet-optimism-deployment stop
```
Clear volumes created by this stack:
To stop all L2 services and also delete chain data:
```bash
# List all relevant volumes
docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"
# Remove all the listed volumes
docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
laconic-so deployment --dir fixturenet-optimism-deployment stop --delete-volumes
```
## Troubleshooting
@@ -5,8 +5,8 @@ repos:
- git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
- git.vdb.to/cerc-io/lighthouse
- github.com/dboreham/foundry
- github.com/ethereum-optimism/optimism@v1.0.4
- github.com/ethereum-optimism/op-geth@v1.101105.2
- github.com/ethereum-optimism/optimism@op-node/v1.3.0
- github.com/ethereum-optimism/op-geth@v1.101304.0
containers:
- cerc/go-ethereum
- cerc/lighthouse
@@ -21,7 +21,7 @@ from stack_orchestrator.deploy.deployer import Deployer, DeployerException, Depl
class DockerDeployer(Deployer):
name: str = "compose"
def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None:
self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name,
compose_env_file=compose_env_file)
+7 -3
View File
@@ -28,6 +28,7 @@ from stack_orchestrator.util import include_exclude_check, get_parsed_stack_conf
from stack_orchestrator.deploy.deployer import Deployer, DeployerException
from stack_orchestrator.deploy.deployer_factory import getDeployer
from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
from stack_orchestrator.deploy.deployment_context import DeploymentContext
from stack_orchestrator.deploy.deployment_create import create as deployment_create
from stack_orchestrator.deploy.deployment_create import init as deployment_init
from stack_orchestrator.deploy.deployment_create import setup as deployment_setup
@@ -56,14 +57,17 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to):
if deploy_to is None:
deploy_to = "compose"
ctx.obj = create_deploy_context(global_options2(ctx), stack, include, exclude, cluster, env_file, deploy_to)
ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to)
# Subcommand is executed now, by the magic of click
def create_deploy_context(global_context, stack, include, exclude, cluster, env_file, deployer):
def create_deploy_context(
global_context, deployment_context: DeploymentContext, stack, include, exclude, cluster, env_file, deployer):
cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file)
deployment_dir = deployment_context.deployment_dir if deployment_context else None
# See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/
deployer = getDeployer(deployer, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster,
deployer = getDeployer(deployer, deployment_dir, compose_files=cluster_context.compose_files,
compose_project_name=cluster_context.cluster,
compose_env_file=cluster_context.env_file)
return DeployCommandContext(stack, cluster_context, deployer)
+6 -8
View File
@@ -13,9 +13,8 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from typing import List
from typing import List, Mapping
from dataclasses import dataclass
from pathlib import Path
from stack_orchestrator.command_types import CommandOptions
from stack_orchestrator.deploy.deployer import Deployer
@@ -38,12 +37,6 @@ class DeployCommandContext:
deployer: Deployer
@dataclass
class DeploymentContext:
deployment_dir: Path
command_context: DeployCommandContext
@dataclass
class VolumeMapping:
host_path: str
@@ -66,3 +59,8 @@ class LaconicStackSetupCommand:
@dataclass
class LaconicStackCreateCommand:
network_dir: str
@dataclass
class DeployEnvVars:
map: Mapping[str, str]
@@ -26,10 +26,10 @@ def getDeployerConfigGenerator(type: str):
print(f"ERROR: deploy-to {type} is not valid")
def getDeployer(type: str, compose_files, compose_project_name, compose_env_file):
def getDeployer(type: str, deployment_dir, compose_files, compose_project_name, compose_env_file):
if type == "compose" or type is None:
return DockerDeployer(compose_files, compose_project_name, compose_env_file)
return DockerDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file)
elif type == "k8s":
return K8sDeployer(compose_files, compose_project_name, compose_env_file)
return K8sDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file)
else:
print(f"ERROR: deploy-to {type} is not valid")
+2 -29
View File
@@ -18,34 +18,7 @@ from pathlib import Path
import sys
from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation
from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context
from stack_orchestrator.deploy.stack import Stack
from stack_orchestrator.deploy.spec import Spec
class DeploymentContext:
dir: Path
spec: Spec
stack: Stack
def get_stack_file(self):
return self.dir.joinpath("stack.yml")
def get_spec_file(self):
return self.dir.joinpath("spec.yml")
def get_env_file(self):
return self.dir.joinpath("config.env")
# TODO: implement me
def get_cluster_name(self):
return None
def init(self, dir):
self.dir = dir
self.stack = Stack()
self.stack.init_from_file(self.get_stack_file())
self.spec = Spec()
self.spec.init_from_file(self.get_spec_file())
from stack_orchestrator.deploy.deployment_context import DeploymentContext
@click.group()
@@ -77,7 +50,7 @@ def make_deploy_context(ctx):
stack_file_path = context.get_stack_file()
env_file = context.get_env_file()
cluster_name = context.get_cluster_name()
return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, cluster_name, env_file,
return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file,
context.spec.obj["deploy-to"])
@@ -0,0 +1,46 @@
# Copyright © 2022, 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from pathlib import Path
from stack_orchestrator.deploy.stack import Stack
from stack_orchestrator.deploy.spec import Spec
class DeploymentContext:
deployment_dir: Path
spec: Spec
stack: Stack
def get_stack_file(self):
return self.deployment_dir.joinpath("stack.yml")
def get_spec_file(self):
return self.deployment_dir.joinpath("spec.yml")
def get_env_file(self):
return self.deployment_dir.joinpath("config.env")
# TODO: implement me
def get_cluster_name(self):
return None
def init(self, dir):
self.deployment_dir = dir
self.spec = Spec()
self.spec.init_from_file(self.get_spec_file())
self.stack = Stack(self.spec.obj["stack"])
self.stack.init_from_file(self.get_stack_file())
+10 -8
View File
@@ -24,8 +24,9 @@ import sys
from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
get_pod_script_paths, get_plugin_code_paths)
from stack_orchestrator.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
from stack_orchestrator.deploy.deployment_context import DeploymentContext
def _make_default_deployment_dir():
@@ -108,8 +109,8 @@ def _fixup_pod_file(pod, spec, compose_dir):
pod["services"][container_name]["ports"] = container_ports
def _commands_plugin_paths(ctx: DeployCommandContext):
plugin_paths = get_plugin_code_paths(ctx.stack)
def _commands_plugin_paths(stack_name: str):
plugin_paths = get_plugin_code_paths(stack_name)
ret = [p.joinpath("deploy", "commands.py") for p in plugin_paths]
return ret
@@ -123,7 +124,7 @@ def call_stack_deploy_init(deploy_command_context):
# Link with the python file in the stack
# Call a function in it
# If no function found, return None
python_file_paths = _commands_plugin_paths(deploy_command_context)
python_file_paths = _commands_plugin_paths(deploy_command_context.stack)
ret = None
init_done = False
@@ -147,7 +148,7 @@ def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetu
# Link with the python file in the stack
# Call a function in it
# If no function found, return None
python_file_paths = _commands_plugin_paths(deploy_command_context)
python_file_paths = _commands_plugin_paths(deploy_command_context.stack)
for python_file_path in python_file_paths:
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
@@ -162,7 +163,7 @@ def call_stack_deploy_create(deployment_context, extra_args):
# Link with the python file in the stack
# Call a function in it
# If no function found, return None
python_file_paths = _commands_plugin_paths(deployment_context.command_context)
python_file_paths = _commands_plugin_paths(deployment_context.stack.name)
for python_file_path in python_file_paths:
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
@@ -311,7 +312,7 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path):
def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
# This function fails with a useful error message if the file doens't exist
parsed_spec = get_parsed_deployment_spec(spec_file)
stack_name = parsed_spec['stack']
stack_name = parsed_spec["stack"]
stack_file = get_stack_file_path(stack_name)
parsed_stack = get_parsed_stack_config(stack_name)
if global_options(ctx).debug:
@@ -367,7 +368,8 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
# stack member here.
deployment_command_context = ctx.obj
deployment_command_context.stack = stack_name
deployment_context = DeploymentContext(Path(deployment_dir), deployment_command_context)
deployment_context = DeploymentContext()
deployment_context.init(Path(deployment_dir))
# Call the deployer to generate any deployer-specific files (e.g. for kind)
deployer_config_generator = getDeployerConfigGenerator(parsed_spec["deploy-to"])
# TODO: make deployment_dir a Path above
+30 -4
View File
@@ -18,7 +18,9 @@ from typing import Any, List, Set
from stack_orchestrator.opts import opts
from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
from stack_orchestrator.deploy.k8s.helpers import parsed_pod_files_map_from_file_names
from stack_orchestrator.deploy.k8s.helpers import parsed_pod_files_map_from_file_names, get_node_pv_mount_path
from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map
from stack_orchestrator.deploy.deploy_types import DeployEnvVars
class ClusterInfo:
@@ -26,11 +28,12 @@ class ClusterInfo:
image_set: Set[str] = set()
app_name: str = "test-app"
deployment_name: str = "test-deployment"
environment_variables: DeployEnvVars
def __init__(self) -> None:
pass
def int_from_pod_files(self, pod_files: List[str]):
def int(self, pod_files: List[str], compose_env_file):
self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
# Find the set of images in the pods
for pod_name in self.parsed_pod_yaml_map:
@@ -42,6 +45,9 @@ class ClusterInfo:
self.image_set.add(image)
if opts.o.debug:
print(f"image_set: {self.image_set}")
self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file))
if (opts.o.debug):
print(f"Env vars: {self.environment_variables.map}")
def get_pvcs(self):
result = []
@@ -50,11 +56,12 @@ class ClusterInfo:
print(f"Volumes: {volumes}")
for volume_name in volumes:
spec = client.V1PersistentVolumeClaimSpec(
storage_class_name="standard",
access_modes=["ReadWriteOnce"],
storage_class_name="manual",
resources=client.V1ResourceRequirements(
requests={"storage": "2Gi"}
)
),
volume_name=volume_name
)
pvc = client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(name=volume_name,
@@ -64,6 +71,24 @@ class ClusterInfo:
result.append(pvc)
return result
def get_pvs(self):
result = []
volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
for volume_name in volumes:
spec = client.V1PersistentVolumeSpec(
storage_class_name="manual",
access_modes=["ReadWriteOnce"],
capacity={"storage": "2Gi"},
host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
)
pv = client.V1PersistentVolume(
metadata=client.V1ObjectMeta(name=volume_name,
labels={"volume-label": volume_name}),
spec=spec,
)
result.append(pv)
return result
# to suit the deployment, and also annotate the container specs to point at said volumes
def get_deployment(self):
containers = []
@@ -78,6 +103,7 @@ class ClusterInfo:
container = client.V1Container(
name=container_name,
image=image,
env=envs_from_environment_variables_map(self.environment_variables.map),
ports=[client.V1ContainerPort(container_port=80)],
volume_mounts=volume_mounts,
resources=client.V1ResourceRequirements(
+21 -6
View File
@@ -30,15 +30,18 @@ class K8sDeployer(Deployer):
k8s_namespace: str = "default"
kind_cluster_name: str
cluster_info : ClusterInfo
deployment_dir: Path
def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None:
if (opts.o.debug):
print(f"Deployment dir: {deployment_dir}")
print(f"Compose files: {compose_files}")
print(f"Project name: {compose_project_name}")
print(f"Env file: {compose_env_file}")
self.deployment_dir = deployment_dir
self.kind_cluster_name = compose_project_name
self.cluster_info = ClusterInfo()
self.cluster_info.int_from_pod_files(compose_files)
self.cluster_info.int(compose_files, compose_env_file)
def connect_api(self):
config.load_kube_config(context=f"kind-{self.kind_cluster_name}")
@@ -47,16 +50,26 @@ class K8sDeployer(Deployer):
def up(self, detach, services):
# Create the kind cluster
# HACK: pass in the config file path here
create_cluster(self.kind_cluster_name, "./test-deployment-dir/kind-config.yml")
create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath("kind-config.yml"))
self.connect_api()
# Ensure the referenced containers are copied into kind
load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
# Create the host-path-mounted PVs for this deployment
pvs = self.cluster_info.get_pvs()
for pv in pvs:
if opts.o.debug:
print(f"Sending this pv: {pv}")
pv_resp = self.core_api.create_persistent_volume(body=pv)
if opts.o.debug:
print("PVs created:")
print(f"{pv_resp}")
# Figure out the PVCs for this deployment
pvcs = self.cluster_info.get_pvcs()
for pvc in pvcs:
if opts.o.debug:
print(f"Sending this: {pvc}")
print(f"Sending this pvc: {pvc}")
pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace)
if opts.o.debug:
print("PVCs created:")
@@ -65,7 +78,7 @@ class K8sDeployer(Deployer):
deployment = self.cluster_info.get_deployment()
# Create the k8s objects
if opts.o.debug:
print(f"Sending this: {deployment}")
print(f"Sending this deployment: {deployment}")
deployment_resp = self.apps_api.create_namespaced_deployment(
body=deployment, namespace=self.k8s_namespace
)
@@ -122,6 +135,8 @@ class K8sDeployerConfigGenerator(DeployerConfigGenerator):
# Check the file isn't already there
# Get the config file contents
content = generate_kind_config(deployment_dir)
if opts.o.debug:
print(f"kind config is: {content}")
config_file = deployment_dir.joinpath(self.config_file_name)
# Write the file
with open(config_file, "w") as output_file:
+33 -4
View File
@@ -14,9 +14,11 @@
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from kubernetes import client
from dotenv import dotenv_values
import os
from pathlib import Path
import subprocess
from typing import Any, Set
from typing import Any, Set, Mapping, List
from stack_orchestrator.opts import opts
from stack_orchestrator.util import get_yaml
@@ -73,6 +75,10 @@ def named_volumes_from_pod_files(parsed_pod_files):
return named_volumes
def get_node_pv_mount_path(volume_name: str):
return f"/mnt/{volume_name}"
def volume_mounts_for_service(parsed_pod_files, service):
result = []
# Find the service
@@ -119,6 +125,14 @@ def _get_host_paths_for_volumes(parsed_pod_files):
return result
def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Path:
if os.path.isabs(data_mount_path):
return data_mount_path
else:
# Python Path voodo that looks pretty odd:
return Path.cwd().joinpath(deployment_dir.joinpath("compose").joinpath(data_mount_path)).resolve()
def parsed_pod_files_map_from_file_names(pod_files):
parsed_pod_yaml_map : Any = {}
for pod_file in pod_files:
@@ -130,9 +144,12 @@ def parsed_pod_files_map_from_file_names(pod_files):
return parsed_pod_yaml_map
def _generate_kind_mounts(parsed_pod_files):
def _generate_kind_mounts(parsed_pod_files, deployment_dir):
volume_definitions = []
volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files)
# Note these paths are relative to the location of the pod files (at present)
# So we need to fix up to make them correct and absolute because kind assumes
# relative to the cwd.
for pod in parsed_pod_files:
parsed_pod_file = parsed_pod_files[pod]
if "services" in parsed_pod_file:
@@ -145,7 +162,8 @@ def _generate_kind_mounts(parsed_pod_files):
# Looks like: test-data:/data
(volume_name, mount_path) = mount_string.split(":")
volume_definitions.append(
f" - hostPath: {volume_host_path_map[volume_name]}\n containerPath: /var/local-path-provisioner"
f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n"
f" containerPath: {get_node_pv_mount_path(volume_name)}"
)
return (
"" if len(volume_definitions) == 0 else (
@@ -177,6 +195,13 @@ def _generate_kind_port_mappings(parsed_pod_files):
)
def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]:
result = []
for env_var, env_val in map.items():
result.append(client.V1EnvVar(env_var, env_val))
return result
# This needs to know:
# The service ports for the cluster
# The bind mounted volumes for the cluster
@@ -201,7 +226,7 @@ def generate_kind_config(deployment_dir: Path):
pod_files = [p for p in compose_file_dir.iterdir() if p.is_file()]
parsed_pod_files_map = parsed_pod_files_map_from_file_names(pod_files)
port_mappings_yml = _generate_kind_port_mappings(parsed_pod_files_map)
mounts_yml = _generate_kind_mounts(parsed_pod_files_map)
mounts_yml = _generate_kind_mounts(parsed_pod_files_map, deployment_dir)
return (
"kind: Cluster\n"
"apiVersion: kind.x-k8s.io/v1alpha4\n"
@@ -210,3 +235,7 @@ def generate_kind_config(deployment_dir: Path):
f"{port_mappings_yml}\n"
f"{mounts_yml}\n"
)
def env_var_map_from_file(file: Path) -> Mapping[str, str]:
return dotenv_values(file)
+3 -2
View File
@@ -20,10 +20,11 @@ from stack_orchestrator.util import get_yaml
class Stack:
name: str
obj: typing.Any
def __init__(self) -> None:
pass
def __init__(self, name: str) -> None:
self.name = name
def init_from_file(self, file_path: Path):
with file_path:
+2
View File
@@ -19,6 +19,7 @@ from stack_orchestrator.command_types import CommandOptions
from stack_orchestrator.repos import setup_repositories
from stack_orchestrator.build import build_containers
from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy import deploy
from stack_orchestrator import version
from stack_orchestrator.deploy import deployment
@@ -48,6 +49,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
cli.add_command(setup_repositories.command, "setup-repositories")
cli.add_command(build_containers.command, "build-containers")
cli.add_command(build_npms.command, "build-npms")
cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
cli.add_command(deploy.command, "deploy-system")
cli.add_command(deployment.command, "deployment")
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
# Note: eventually this test should be folded into ../deploy/
# but keeping it separate for now for convenience
TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
# Dump environment variables for debugging
echo "Environment variables:"
env
# Set a non-default repo dir
export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
echo "Testing this package: $TEST_TARGET_SO"
echo "Test version command"
reported_version_string=$( $TEST_TARGET_SO version )
echo "Version reported is: ${reported_version_string}"
echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
rm -rf $CERC_REPO_BASE_DIR
mkdir -p $CERC_REPO_BASE_DIR
# Test basic stack-orchestrator deploy
test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
$TEST_TARGET_SO --stack test deploy --deploy-to k8s init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
# Check the file now exists
if [ ! -f "$test_deployment_spec" ]; then
echo "deploy init test: spec file not present"
echo "deploy init test: FAILED"
exit 1
fi
echo "deploy init test: passed"
$TEST_TARGET_SO --stack test deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
# Check the deployment dir exists
if [ ! -d "$test_deployment_dir" ]; then
echo "deploy create test: deployment directory not present"
echo "deploy create test: FAILED"
exit 1
fi
echo "deploy create test: passed"
# Check the file writted by the create command in the stack now exists
if [ ! -f "$test_deployment_dir/create-file" ]; then
echo "deploy create test: create output file not present"
echo "deploy create test: FAILED"
exit 1
fi
# And has the right content
create_file_content=$(<$test_deployment_dir/create-file)
if [ ! "$create_file_content" == "create-command-output-data" ]; then
echo "deploy create test: create output file contents not correct"
echo "deploy create test: FAILED"
exit 1
fi
echo "deploy create output file test: passed"
# Try to start the deployment
$TEST_TARGET_SO deployment --dir $test_deployment_dir start
# TODO: add a check to see if the container is up
# Sleep because k8s not up yet
sleep 30
# Check logs command works
log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
echo "deployment logs test: passed"
else
echo "deployment logs test: FAILED"
exit 1
fi
# Check the config variable CERC_TEST_PARAM_1 was passed correctly
if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
echo "deployment config test: passed"
else
echo "deployment config test: FAILED"
exit 1
fi
# Stop and clean up
$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
echo "Test passed"
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
# Dump environment variables for debugging
echo "Environment variables:"
env
# Test basic stack-orchestrator webapp
echo "Running stack-orchestrator webapp test"
# Bit of a hack, test the most recent package
TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
# Set a non-default repo dir
export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
echo "Testing this package: $TEST_TARGET_SO"
echo "Test version command"
reported_version_string=$( $TEST_TARGET_SO version )
echo "Version reported is: ${reported_version_string}"
echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
rm -rf $CERC_REPO_BASE_DIR
mkdir -p $CERC_REPO_BASE_DIR
git clone https://git.vdb.to/cerc-io/test-progressive-web-app.git $CERC_REPO_BASE_DIR/test-progressive-web-app
# Test webapp command execution
$TEST_TARGET_SO build-webapp --source-repo $CERC_REPO_BASE_DIR/test-progressive-web-app
UUID=`uuidgen`
set +e
CONTAINER_ID=$(docker run -p 3000:3000 -d cerc/test-progressive-web-app:local)
sleep 3
wget -O test.before -m http://localhost:3000
docker remove -f $CONTAINER_ID
CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$UUID -d cerc/test-progressive-web-app:local)
sleep 3
wget -O test.after -m http://localhost:3000
docker remove -f $CONTAINER_ID
echo "###########################################################################"
echo ""
grep "$UUID" test.before > /dev/null
if [ $? -ne 1 ]; then
echo "BEFORE: FAILED"
exit 1
else
echo "BEFORE: PASSED"
fi
grep "$UUID" test.after > /dev/null
if [ $? -ne 0 ]; then
echo "AFTER: FAILED"
exit 1
else
echo "AFTER: PASSED"
fi
exit 0