Compare commits

..
Author SHA1 Message Date
Zach 88db2ff766 Update stack.yml 2023-11-28 12:10:03 -05:00
zramsay 5f556e127a update osmosis readme 2023-11-15 13:24:52 -05:00
zramsay 926997b21c updates 2023-11-14 20:26:26 +00:00
zramsay 0d91a62f84 fix for neww stack format 2023-11-14 19:57:22 +00:00
zramsay 7d27eaef0f dont use 3000 2023-11-13 13:35:09 -05:00
zramsay 8ad2b692ec chmod 2023-11-13 13:35:09 -05:00
zramsay 54cc993fa4 osmosis FE stack 2023-11-13 13:35:07 -05:00
23 changed files with 109 additions and 245 deletions
-54
View File
@@ -1,54 +0,0 @@
### Building and Running Webapps
It is possible to build and run Next.js webapps using the `build-webapp` and `run-webapp` subcommands.
To make it easier to build once and deploy into different environments and with different configuration,
compilation and static page generation are separated in the `build-webapp` and `run-webapp` steps.
This offers much more flexibilty than standard Next.js build methods, since any environment variables accessed
via `process.env`, whether for pages or for API, will have values drawn from their runtime deployment environment,
not their build environment.
## Building
Building usually requires no additional configuration. By default, the Next.js version specified in `package.json`
is used, and either `yarn` or `npm` will be used automatically depending on which lock files are present. These
can be overidden with the build arguments `CERC_NEXT_VERSION` and `CERC_BUILD_TOOL` respectively. For example: `--extra-build-args "--build-arg CERC_NEXT_VERSION=13.4.12"`
**Example**:
```
$ cd ~/cerc
$ git clone git@git.vdb.to:cerc-io/test-progressive-web-app.git
$ laconic-so build-webapp --source-repo ~/cerc/test-progressive-web-app
...
Built host container for ~/cerc/test-progressive-web-app with tag:
cerc/test-progressive-web-app:local
To test locally run:
laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment.env
```
## Running
With `run-webapp` a new container will be launched with runtime configuration provided by `--env-file` (if specified) and published on an available port. Multiple instances can be launched with different configuration.
**Example**:
```
# Production env
$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/production.env
Image: cerc/test-progressive-web-app:local
ID: 4c6e893bf436b3e91a2b92ce37e30e499685131705700bd92a90d2eb14eefd05
URL: http://localhost:32768
# Dev env
$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/dev.env
Image: cerc/test-progressive-web-app:local
ID: 9ab96494f563aafb6c057d88df58f9eca81b90f8721a4e068493a289a976051c
URL: http://localhost:32769
```
+7 -25
View File
@@ -27,7 +27,7 @@ import subprocess
import click
import importlib.resources
from pathlib import Path
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
from stack_orchestrator.base import get_npm_registry_url
# TODO: find a place for this
@@ -58,8 +58,7 @@ def make_container_build_env(dev_root_path: str,
return container_build_env
def process_container(stack: str,
container,
def process_container(container,
container_build_dir: str,
container_build_env: dict,
dev_root_path: str,
@@ -70,29 +69,12 @@ def process_container(stack: str,
):
if not quiet:
print(f"Building: {container}")
default_container_tag = f"{container}:local"
container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
# Check if this is in an external stack
if stack_is_external(stack):
container_parent_dir = Path(stack).joinpath("container-build")
temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
temp_build_script_filename = temp_build_dir.joinpath("build.sh")
# Now check if the container exists in the external stack.
if not temp_build_script_filename.exists():
# If not, revert to building an internal container
container_parent_dir = container_build_dir
else:
container_parent_dir = container_build_dir
build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
build_script_filename = build_dir.joinpath("build.sh")
build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
build_script_filename = os.path.join(build_dir, "build.sh")
if verbose:
print(f"Build script filename: {build_script_filename}")
if os.path.exists(build_script_filename):
build_command = build_script_filename.as_posix()
build_command = build_script_filename
else:
if verbose:
print(f"No script file found: {build_script_filename}, using default build script")
@@ -102,7 +84,7 @@ def process_container(stack: str,
repo_full_path = os.path.join(dev_root_path, repo_dir)
repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
build_command = os.path.join(container_build_dir,
"default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
"default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
if not dry_run:
if verbose:
print(f"Executing: {build_command} with environment: {container_build_env}")
@@ -176,7 +158,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
for container in containers_in_scope:
if include_exclude_check(container, include, exclude):
process_container(stack, container, container_build_dir, container_build_env,
process_container(container, container_build_dir, container_build_env,
dev_root_path, quiet, verbose, dry_run, continue_on_error)
else:
if verbose:
+2 -2
View File
@@ -60,7 +60,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
force_rebuild, extra_build_args)
build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
verbose, dry_run, continue_on_error)
@@ -73,5 +73,5 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
verbose, dry_run, continue_on_error)
-16
View File
@@ -1,16 +0,0 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
stack_file_name = "stack.yml"
@@ -2,7 +2,7 @@ version: "3.2"
# See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up
services:
ipfs:
image: ipfs/kubo:master-2023-02-20-714a968
image: ipfs/kubo:v0.24.0
restart: always
volumes:
- ./ipfs/import:/import
@@ -0,0 +1,8 @@
version: "3.2"
services:
osmosis-front-end:
image: cerc/osmosis-front-end:local
restart: always
ports:
- "3002:3002" #TODO make `3000` when using the deployment feature
@@ -39,4 +39,4 @@ EXPOSE 3000
COPY /scripts /scripts
# Default command sleeps forever so docker doesn't kill it
ENTRYPOINT ["/scripts/start-serving-app.sh"]
CMD ["/scripts/start-serving-app.sh"]
@@ -11,19 +11,3 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}
docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
cat <<EOF
#################################################################
Built host container for $CERC_CONTAINER_BUILD_WORK_DIR with tag:
$CERC_CONTAINER_BUILD_TAG
To test locally run:
laconic-so run-webapp --image $CERC_CONTAINER_BUILD_TAG --env-file /path/to/environment.env
EOF
fi
@@ -38,9 +38,6 @@ for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'
orig_name=$(echo -n "${e}" | sed 's/"//g')
cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
cur_val=$(echo -n "\$${cur_name}" | envsubst)
if [ "$CERC_RETAIN_ENV_QUOTES" != "true" ]; then
cur_val=$(sed "s/^[\"']//" <<< "$cur_val" | sed "s/[\"']//")
fi
esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
echo "$f: $cur_name=$cur_val"
sed -i "s/$orig_name/$esc_val/g" $f
@@ -3,16 +3,7 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60}
tpid=""
ctrl_c() {
kill $tpid $(ps -ef | grep node | grep next | awk '{print $2}') 2>/dev/null
}
trap ctrl_c INT
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
@@ -34,27 +25,18 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
jq -e '.scripts.cerc_generate' package.json >/dev/null
if [ $? -eq 0 ]; then
npm run cerc_generate > gen.out 2>&1 &
tail -f gen.out &
tpid=$!
tail -n0 -f gen.out | sed '/rendered as static HTML/ q'
count=0
generate_done="false"
while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do
while [ $count -lt 10 ]; do
sleep 1
count=$((count + 1))
grep 'rendered as static HTML' gen.out > /dev/null
if [ $? -eq 0 ]; then
generate_done="true"
ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null
if [ $? -ne 0 ]; then
break
else
count=$((count + 1))
fi
done
if [ $generate_done != "true" ]; then
echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2
exit 1
fi
kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
tpid=""
kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
fi
fi
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Build the osmosis front end image
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
docker build -t cerc/osmosis-front-end:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend
@@ -2,7 +2,7 @@ version: "1.0"
name: merkl-sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.2
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.1
containers:
- cerc/watcher-merkl-sushiswap-v3
pods:
@@ -0,0 +1,35 @@
# self-hosted osmosis
Build and deploy:
- 1) self-hosted gitea,
- 2) an ipfs node,
- 3) the osmosis front end,
- 4) a laconicd chain
```
# support image for the gitea package registry
laconic-so --stack build-support build-containers
# todo: pre-run clone
# clones and builds several things
laconic-so --stack osmosis setup-repositories
laconic-so --stack osmosis build-containers
laconic-so --stack osmosis deploy up
```
Setup a test chain:
```
export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
laconic-so --stack fixturenet-laconic-loaded build-containers
export LACONIC_HOSTED_ENDPOINT=http://<your-IP>
laconic-so --stack fixturenet-laconic-loaded deploy up
```
then `docker exec` into the `laconicd` container and either export the private key or create a new one and send funds to it. Use that private key for `LACONIC_HOTWALLET_KEY`.
@@ -0,0 +1,27 @@
version: "0.1"
name: osmosis
repos:
# these are for gitea
- git.vdb.to/cerc-io/hosting@names-for-so
- gitea.com/gitea/act_runner
# add the osmosis FE
- github.com/osmosis-labs/osmosis-frontend
containers:
- cerc/act-runner
- cerc/act-runner-task-executor
# note: osmosis builds but doesn't run
- cerc/osmosis-front-end
pods:
- name: gitea
repository: cerc-io/hosting
path: gitea
pre_start_command: "run-this-first.sh"
post_start_command: "initialize-gitea.sh"
# todo, e.g., mirroring all of osmosis repos: https://git.vdb.to/cerc-io/hosting/pulls/42
- name: act-runner
repository: cerc-io/hosting
path: act-runner
pre_start_command: "pre_start.sh"
post_start_command: "post_start.sh"
- osmosis-front-end
- kubo
@@ -2,7 +2,7 @@ version: "1.0"
name: sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.2
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.1
containers:
- cerc/watcher-sushiswap-v3
pods:
@@ -61,10 +61,9 @@ class DockerDeployer(Deployer):
except DockerException as e:
raise DeployerException(e)
def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
def run(self, image, command, user, volumes, entrypoint=None):
try:
return self.docker.run(image=image, command=command, user=user, volumes=volumes,
entrypoint=entrypoint, envs=env, detach=detach, publish_all=True)
return self.docker.run(image=image, command=command, user=user, volumes=volumes, entrypoint=entrypoint)
except DockerException as e:
raise DeployerException(e)
+1 -1
View File
@@ -44,7 +44,7 @@ class Deployer(ABC):
pass
@abstractmethod
def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
def run(self, image, command, user, volumes, entrypoint):
pass
+1 -1
View File
@@ -120,7 +120,7 @@ class K8sDeployer(Deployer):
log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
return log_stream_from_string(log_data)
def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
def run(self, image, command, user, volumes, entrypoint=None):
# We need to figure out how to do this -- check why we're being called first
pass
-59
View File
@@ -1,59 +0,0 @@
# Copyright © 2022, 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
# Builds webapp containers
# env vars:
# CERC_REPO_BASE_DIR defaults to ~/cerc
# TODO: display the available list of containers; allow re-build of either all or specific containers
import hashlib
import click
from dotenv import dotenv_values
from stack_orchestrator.deploy.deployer_factory import getDeployer
@click.command()
@click.option("--image", help="image to deploy", required=True)
@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')")
@click.option("--env-file", help="environment file for webapp")
@click.pass_context
def command(ctx, image, deploy_to, env_file):
'''build the specified webapp container'''
env = {}
if env_file:
env = dotenv_values(env_file)
unique_cluster_descriptor = f"{image},{env}"
hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
cluster = f"laconic-webapp-{hash}"
deployer = getDeployer(deploy_to,
deployment_dir=None,
compose_files=None,
compose_project_name=cluster,
compose_env_file=None)
container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True)
# Make configurable?
webappPort = "3000/tcp"
# TODO: This assumes a Docker container object...
if webappPort in container.network_settings.ports:
mapping = container.network_settings.ports[webappPort][0]
print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""")
-2
View File
@@ -20,7 +20,6 @@ from stack_orchestrator.repos import setup_repositories
from stack_orchestrator.build import build_containers
from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy import run_webapp
from stack_orchestrator.deploy import deploy
from stack_orchestrator import version
from stack_orchestrator.deploy import deployment
@@ -51,7 +50,6 @@ cli.add_command(setup_repositories.command, "setup-repositories")
cli.add_command(build_containers.command, "build-containers")
cli.add_command(build_npms.command, "build-npms")
cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(run_webapp.command, "run-webapp")
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
cli.add_command(deploy.command, "deploy-system")
cli.add_command(deployment.command, "deployment")
+6 -14
View File
@@ -25,8 +25,7 @@ import click
import importlib.resources
from pathlib import Path
import yaml
from stack_orchestrator.constants import stack_file_name
from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
from stack_orchestrator.util import include_exclude_check
class GitProgress(git.RemoteProgress):
@@ -239,20 +238,13 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
repos_in_scope = []
if stack:
if stack_is_external(stack):
stack_file_path = Path(stack).joinpath(stack_file_name)
else:
# In order to be compatible with Python 3.8 we need to use this hack to get the path:
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name)
if not stack_file_path.exists():
error_exit(f"stack {stack} does not exist")
# In order to be compatible with Python 3.8 we need to use this hack to get the path:
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml")
with stack_file_path:
stack_config = yaml.safe_load(open(stack_file_path, "r"))
if "repos" not in stack_config:
error_exit(f"stack {stack} does not define any repositories")
else:
repos_in_scope = stack_config["repos"]
# TODO: syntax check the input here
repos_in_scope = stack_config['repos']
else:
repos_in_scope = all_repos
-11
View File
@@ -150,12 +150,6 @@ def get_parsed_deployment_spec(spec_file):
sys.exit(1)
def stack_is_external(stack: str):
# Bit of a hack: if the supplied stack string represents
# a path that exists then we assume it must be external
return Path(stack).exists() if stack is not None else False
def get_yaml():
# See: https://stackoverflow.com/a/45701840/1701505
yaml = ruamel.yaml.YAML()
@@ -173,8 +167,3 @@ def global_options(ctx):
# TODO: hack
def global_options2(ctx):
return ctx.parent.obj
def error_exit(s):
print(f"ERROR: {s}")
sys.exit(1)
+4 -8
View File
@@ -1,10 +1,8 @@
#!/usr/bin/env bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
# Dump environment variables for debugging
echo "Environment variables:"
env
@@ -30,18 +28,16 @@ CHECK="SPECIAL_01234567890_TEST_STRING"
set +e
CONTAINER_ID=$(docker run -p 3000:3000 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
CONTAINER_ID=$(docker run -p 3000:3000 -d cerc/test-progressive-web-app:local)
sleep 3
wget -t 7 -O test.before -m http://localhost:3000
wget -O test.before -m http://localhost:3000
docker logs $CONTAINER_ID
docker remove -f $CONTAINER_ID
CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -d cerc/test-progressive-web-app:local)
sleep 3
wget -t 7 -O test.after -m http://localhost:3000
wget -O test.after -m http://localhost:3000
docker logs $CONTAINER_ID
docker remove -f $CONTAINER_ID
echo "###########################################################################"