Compare commits

...
Author SHA1 Message Date
dboreham c465153cc7 Fixes for repos checked out from a tag rather than a branch (#437)
* Fixes for repos checked out from a tag rather than a branch

* Fix happy path
2023-06-24 11:19:20 -06:00
Zach c9bb0d0ccb Lasso stack (#434)
* lasso stack

* add readme

* copy pasta
2023-06-23 14:27:05 -04:00
dboreham c94fa3cccf Use cerc npm registry as default (#432) 2023-06-22 06:36:05 -06:00
Zach 55af33e0e8 Merge pull request #427 from cerc-io/fantom
Fantom
2023-06-20 17:24:38 -04:00
zramsay c0f07c2d4f typo 2023-06-20 11:21:26 -04:00
Zach 7ca7bcc952 Cloud init scripts for user/dev mode (#430)
* cloud init install

* add dev mode script + description

* instructions
2023-06-20 10:09:30 -04:00
zramsay ee8c8a0c13 add fantom docs 2023-06-20 10:02:14 -04:00
iskay 30ce3deb31 update build script for rc.5 2023-06-16 18:39:59 +00:00
Ian a6cf131cd3 add network mode "host" 2023-06-13 12:19:23 -04:00
zramsay 25b13d7822 doc fixes 2023-06-12 21:00:30 -04:00
iskay 01500e78b6 initial node 2023-06-12 20:54:39 -04:00
iskay 4ecc98dd46 simple node 2023-06-12 20:54:39 -04:00
nabarun 32f8d65bb8 Update mobymask-v2 stack with lighthouse-cli and branch checkout feature (#425)
* Update optimism stack yml for lighthouse-cli

* Use branch checkout feature in mobymask stack
2023-06-07 18:48:59 +05:30
26 changed files with 299 additions and 84 deletions
+8
View File
@@ -16,6 +16,7 @@
import os
from abc import ABC, abstractmethod
from .deploy_system import get_stack_status
from decouple import config
def get_stack(config, stack):
@@ -69,3 +70,10 @@ class package_registry_stack(base_stack):
def get_url(self):
return self.url
def get_npm_registry_url():
# If an auth token is not defined, we assume the default should be the cerc registry
# If an auth token is defined, we assume the local gitea should be used.
default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config("CERC_NPM_AUTH_TOKEN", default=None) else "https://git.vdb.to/api/packages/cerc-io/npm/"
return config("CERC_NPM_REGISTRY_URL", default=default_npm_registry_url)
+2 -1
View File
@@ -28,6 +28,7 @@ import click
import importlib.resources
from pathlib import Path
from .util import include_exclude_check, get_parsed_stack_config
from .base import get_npm_registry_url
# TODO: find a place for this
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -84,7 +85,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
# TODO: make this configurable
container_build_env = {
"CERC_NPM_REGISTRY_URL": config("CERC_NPM_REGISTRY_URL", default="http://gitea.local:3000/api/packages/cerc-io/npm/"),
"CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
"CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""),
"CERC_REPO_BASE_DIR": dev_root_path,
"CERC_CONTAINER_BASE_DIR": container_build_dir,
@@ -0,0 +1,8 @@
version: "3.2"
services:
lasso:
image: cerc/lasso:local
restart: always
ports:
- "0.0.0.0:3000:3000"
@@ -0,0 +1,17 @@
version: "3.8"
services:
go-opera:
restart: unless-stopped
image: cerc/go-opera:local
entrypoint: ["sh", "/docker-entrypoint-scripts.d/start-node.sh"]
volumes:
- ../config/mainnet-go-opera/start-node.sh:/docker-entrypoint-scripts.d/start-node.sh
# TODO: ports taken from dockerfile, determine which are needed
ports:
- "5050:5050" # p2p port, needed
- "5050:5050/udp"
- "18545:18545" # http rpc port
- "18546:18546" # websockets rpc port
#- "18547" # unknown
#- "19090" # unknown
@@ -1,8 +1,9 @@
version: '3.2'
services:
# Builds and serves the peer-test react-app
peer-test-app:
# Builds and serves the peer-test react-app
restart: unless-stopped
image: cerc/react-peer:local
working_dir: /scripts
env_file:
@@ -0,0 +1 @@
@@ -0,0 +1,7 @@
#!/bin/bash
# download genesis file
wget https://download.fantom.network/mainnet-109331-no-history.g
./opera --genesis=mainnet-109331-no-history.g --db.preset ldb-1 --syncmode snap --http --http.addr="0.0.0.0" --http.corsdomain="*" --http.api=eth,web3,net,txpool,ftm --ws --ws.addr="0.0.0.0" --ws.origins="*" --ws.api=eth,web3,net,txpool,ftm --cache 8192
#tail -f /dev/null
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# Build cerc/go-opera
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
# Checkout appropriate release; refer to https://docs.fantom.foundation/
OPERA_TAG=${OPERA_TAG:-release/1.1.2-rc.5}
git -C ${CERC_REPO_BASE_DIR}/go-opera checkout ${OPERA_TAG}
# Repo's dockerfile gives build error because it's hardcoded for go 1.17; go 1.19 is required
sed -i 's/FROM golang:1\.[0-9]*-alpine as builder/FROM golang:1.19-alpine as builder/' ${CERC_REPO_BASE_DIR}/go-opera/docker/Dockerfile.opera
docker build -f ${CERC_REPO_BASE_DIR}/go-opera/docker/Dockerfile.opera -t cerc/go-opera:local ${build_command_args} ${CERC_REPO_BASE_DIR}/go-opera
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Build the lasso image
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
docker build -t cerc/lasso:local -f ${CERC_REPO_BASE_DIR}/lasso/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/lasso
+2
View File
@@ -41,3 +41,5 @@ cerc/watcher-azimuth
cerc/ipld-eth-state-snapshot
cerc/watcher-gelato
cerc/lotus
cerc/go-opera
cerc/lasso
+2
View File
@@ -28,3 +28,5 @@ fixturenet-pocket
watcher-azimuth
watcher-gelato
fixturenet-lotus
mainnet-go-opera
lasso
+2
View File
@@ -34,3 +34,5 @@ github.com/cerc-io/ipld-eth-state-snapshot
github.com/cerc-io/gelato-watcher-ts
github.com/filecoin-project/lotus
git.vdb.to/cerc-io/test-project
github.com/Fantom-foundation/go-opera
github.com/cerc-io/lasso
@@ -14,14 +14,6 @@ laconic-so --stack fixturenet-optimism setup-repositories
# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
```
Checkout to the required versions and branches in repos:
```bash
# Optimism
cd ~/cerc/optimism
git checkout v1.0.4
```
Build the container images:
```bash
@@ -14,14 +14,6 @@ laconic-so --stack fixturenet-optimism setup-repositories --exclude github.com/c
# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
```
Checkout to the required versions and branches in repos:
```bash
# Optimism
cd ~/cerc/optimism
git checkout v1.0.4
```
Build the container images:
```bash
@@ -3,12 +3,14 @@ name: fixturenet-optimism
decription: "Optimism Fixturenet"
repos:
- github.com/cerc-io/go-ethereum
- github.com/cerc-io/lighthouse
- github.com/dboreham/foundry
- github.com/ethereum-optimism/optimism
- github.com/ethereum-optimism/op-geth
- github.com/ethereum-optimism/optimism@v1.0.4
- github.com/ethereum-optimism/op-geth@v1.101105.2
containers:
- cerc/go-ethereum
- cerc/lighthouse
- cerc/lighthouse-cli
- cerc/fixturenet-eth-geth
- cerc/fixturenet-eth-lighthouse
- cerc/foundry
+7
View File
@@ -0,0 +1,7 @@
# lasso
```
laconic-so --stack lasso setup-repositories
laconic-so --stack lasso build-containers
laconic-so --stack lasso deploy up
```
+8
View File
@@ -0,0 +1,8 @@
version: "0.1"
name: lasso
repos:
- github.com/cerc-io/lasso
containers:
- cerc/lasso
pods:
- lasso
@@ -0,0 +1,88 @@
# Opera (Fantom)
Deploy a Fantom API node.
## Clone required repositories
```
$ laconic-so --stack mainnet-go-opera setup-repositories
```
## Build the fixturenet-eth containers
```
$ laconic-so --stack mainnet-go-opera build-containers
```
## Deploy the stack
```
$ laconic-so --stack mainnet-go-opera deploy up
```
## Check logs
```
$ laconic-so --stack mainnet-go-opera deploy logs
```
You'll see something like:
```
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | Connecting to download.fantom.network (65.108.45.88:443)
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | saving to 'mainnet-109331-no-history.g'
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | mainnet-109331-no-hi 100% |********************************| 16326 0:00:00 ETA
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | 'mainnet-109331-no-history.g' saved
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.034] Maximum peer count total=50
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.034] Smartcard socket not found, disabling err="stat /run/pcscd/pcscd.comm: no such file or directory"
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.034] Genesis file is a known preset name="Mainnet-109331 without history"
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.052] Applying genesis state
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.052] - Reading epochs unit 0
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.054] - Reading blocks unit 0
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.530] Applied genesis state name=main id=250 genesis=0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.531] Regenerated local transaction journal transactions=0 accounts=0
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.532] Starting peer-to-peer node instance=go-opera/v1.1.2-rc.5-50cd051d-1677276206/linux-amd64/go1.19.10
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.536] New local node record seq=1 id=5e40f984908317cd ip=127.0.0.1 udp=5050 tcp=5050
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.537] Started P2P networking self=enode://3ffb15988ca5a79b63dbe48be89d9d8b48dc4845d318fe08231a0ab49d3b23476e2561044311dc257405f882f7c52ff7b128c8bd1b6d85cf7205a6fed6555443@127.0.0.1:5050
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.537] IPC endpoint opened url=/root/.opera/opera.ipc
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] HTTP server started endpoint=[::]:18545 prefix= cors=* vhosts=localhost
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] WebSocket enabled url=ws://[::]:18546
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] Rebuilding state snapshot
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] EVM snapshot module=gossip-store at=000000..000000 generating=true
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] Resuming state snapshot generation accounts=0 slots=0 storage=0.00B elapsed="189.74µs"
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] Generated state snapshot accounts=0 slots=0 storage=0.00B elapsed="265.061µs"
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:34.788] New LLR summary last_epoch=0 last_block=37676611 new_evs=0 new_ers=0 new_bvs=64 new_brs=0 age=none
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:35.040] New local node record seq=2 id=5e40f984908317cd ip=186.233.184.56 udp=5050 tcp=5050
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:42.788] New LLR summary last_epoch=114604 last_block=37753891 new_evs=24581 new_ers=5272 new_bvs=233257 new_brs=780 age=1y1mo5d
laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:50.827] New LLR summary last_epoch=115574 last_block=38118749 new_evs=4907 new_ers=971 new_bvs=1098760 new_brs=3768 age=1y1mo2d
```
Consecutive lines of "New LLR summary" shows that your node is sync'ing.
## Use the opera admin console
```
$ docker exec -it $(docker ps -q --filter "name=go-opera") /bin/sh
```
then:
```
$ ./opera attach
```
and check the node info:
```
> admin.nodeInfo
```
Run `exit` twice to return to your terminal.
## Clean up
Stop all services running in the background:
```bash
$ laconic-so --stack mainnet-go-opera deploy down
```
@@ -0,0 +1,9 @@
version: "1.1"
name: mainnet-opera
decription: "Fantom mainnet node"
repos:
- github.com/Fantom-foundation/go-opera
containers:
- cerc/go-opera
pods:
- mainnet-go-opera
-20
View File
@@ -18,26 +18,6 @@ laconic-so --stack mobymask-v2 setup-repositories
NOTE: If repositories already exist and are checked out to different versions, `setup-repositories` command will throw an error.
For getting around this, the repositories mentioned below can be removed and then run the command.
Checkout to the required versions and branches in repos
```bash
# watcher-ts
cd ~/cerc/watcher-ts
git checkout v0.2.41
# mobymask-v2-watcher-ts
cd ~/cerc/mobymask-v2-watcher-ts
git checkout v0.1.1
# MobyMask
cd ~/cerc/MobyMask
git checkout v0.1.2
# Optimism
cd ~/cerc/optimism
git checkout v1.0.4
```
Build the container images:
```bash
@@ -14,22 +14,6 @@ laconic-so --stack mobymask-v2 setup-repositories --include github.com/cerc-io/M
# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
```
Checkout to the required versions and branches in repos:
```bash
# watcher-ts
cd ~/cerc/watcher-ts
git checkout v0.2.41
# mobymask-v2-watcher-ts
cd ~/cerc/mobymask-v2-watcher-ts
git checkout v0.1.1
# MobyMask
cd ~/cerc/MobyMask
git checkout v0.1.2
```
Build the container images:
```bash
+8 -5
View File
@@ -2,21 +2,24 @@ version: "1.0"
name: mobymask-v2
repos:
- github.com/cerc-io/go-ethereum
- github.com/cerc-io/lighthouse
- github.com/dboreham/foundry
- github.com/ethereum-optimism/optimism
- github.com/ethereum-optimism/op-geth
- github.com/cerc-io/watcher-ts
- github.com/cerc-io/mobymask-v2-watcher-ts
- github.com/cerc-io/MobyMask
- github.com/ethereum-optimism/optimism@v1.0.4
- github.com/ethereum-optimism/op-geth@v1.101105.2
- github.com/cerc-io/watcher-ts@v0.2.43
- github.com/cerc-io/mobymask-v2-watcher-ts@v0.1.1
- github.com/cerc-io/MobyMask@v0.1.2
containers:
- cerc/go-ethereum
- cerc/lighthouse
- cerc/lighthouse-cli
- cerc/fixturenet-eth-geth
- cerc/fixturenet-eth-lighthouse
- cerc/foundry
- cerc/optimism-contracts
- cerc/optimism-l2geth
- cerc/optimism-op-batcher
- cerc/optimism-op-proposer
- cerc/optimism-op-node
- cerc/watcher-ts
- cerc/watcher-mobymask-v2
@@ -30,22 +30,6 @@ Clone required repositories:
# 100%|##############################################################################################################################################| 1.41k/1.41k [00:18<00:00, 76.4B/s]
```
Checkout to the required versions and branches in repos:
```bash
# watcher-ts
cd ~/cerc/watcher-ts
git checkout v0.2.41
# mobymask-v2-watcher-ts
cd ~/cerc/mobymask-v2-watcher-ts
git checkout v0.1.1
# MobyMask
cd ~/cerc/MobyMask
git checkout v0.1.2
```
Build the container images:
```bash
+29 -7
View File
@@ -69,8 +69,26 @@ def host_and_path_for_repo(fully_qualified_repo):
return repo_host_split[0], "/".join(repo_host_split[1:]), repo_branch
# See: https://stackoverflow.com/questions/18659425/get-git-current-branch-tag-name
def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
current_repo_branch_or_tag = "***UNDETERMINED***"
is_branch = False
try:
current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).active_branch.name
is_branch = True
except TypeError as error:
# This means that the current ref is not a branch, so possibly a tag
# Let's try to get the tag
current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
# Note that git is assymetric -- the tag you told it to check out may not be the one
# you get back here (if there are multiple tags associated with the same commit)
return current_repo_branch_or_tag, is_branch
# TODO: fix the messy arg list here
def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo):
if verbose:
print(f"Processing repo: {fully_qualified_repo}")
repo_host, repo_path, repo_branch = host_and_path_for_repo(fully_qualified_repo)
git_ssh_prefix = f"git@{repo_host}:"
git_http_prefix = f"https://{repo_host}/"
@@ -78,9 +96,9 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
repoName = repo_path.split("/")[-1]
full_filesystem_repo_path = os.path.join(dev_root_path, repoName)
is_present = os.path.isdir(full_filesystem_repo_path)
current_repo_branch = git.Repo(full_filesystem_repo_path).active_branch.name if is_present else None
(current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(full_filesystem_repo_path) if is_present else (None, None)
if not quiet:
present_text = f"already exists active branch: {current_repo_branch}" if is_present \
present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
else 'Needs to be fetched'
print(f"Checking: {full_filesystem_repo_path}: {present_text}")
# Quick check that it's actually a repo
@@ -93,9 +111,12 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
if verbose:
print(f"Running git pull for {full_filesystem_repo_path}")
if not check_only:
git_repo = git.Repo(full_filesystem_repo_path)
origin = git_repo.remotes.origin
origin.pull(progress=None if quiet else GitProgress())
if is_branch:
git_repo = git.Repo(full_filesystem_repo_path)
origin = git_repo.remotes.origin
origin.pull(progress=None if quiet else GitProgress())
else:
print(f"skipping pull because this repo checked out a tag")
else:
print("(git pull skipped)")
if not is_present:
@@ -122,14 +143,15 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
branch_to_checkout = repo_branch
if branch_to_checkout:
if current_repo_branch is None or (current_repo_branch and (current_repo_branch != branch_to_checkout)):
if current_repo_branch_or_tag is None or (current_repo_branch_or_tag and (current_repo_branch_or_tag != branch_to_checkout)):
if not quiet:
print(f"switching to branch {branch_to_checkout} in repo {repo_path}")
git_repo = git.Repo(full_filesystem_repo_path)
# git checkout works for both branches and tags
git_repo.git.checkout(branch_to_checkout)
else:
if verbose:
print(f"repo {repo_path} is already switched to branch {branch_to_checkout}")
print(f"repo {repo_path} is already on branch/tag {branch_to_checkout}")
def parse_branches(branches_string):
+44
View File
@@ -0,0 +1,44 @@
#cloud-config
# Used for easily testing stacks-in-development on cloud platforms
# Assumes Ubuntu, edit the last line if targeting a different OS
# Once SSH'd into the server, run:
# `$ cd stack-orchestrator`
# `$ git checkout <branch>
# `$ ./scripts/developer-mode-setup.sh`
# `$ source ./venv/bin/activate`
# Followed by the stack instructions.
package_update: true
package_upgrade: true
groups:
- docker
system_info:
default_user:
groups: [ docker ]
packages:
- apt-transport-https
- ca-certificates
- curl
- jq
- git
- gnupg
- lsb-release
- unattended-upgrades
- python3.10-venv
- pip
runcmd:
- mkdir -p /etc/apt/keyrings
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
- echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
- apt-get update
- apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
- systemctl enable docker
- systemctl start docker
- git clone https://github.com/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator
+35
View File
@@ -0,0 +1,35 @@
#cloud-config
# Used for installing Stack Orchestrator on platforms that support `cloud-init`
# Tested on Ubuntu
package_update: true
package_upgrade: true
groups:
- docker
system_info:
default_user:
groups: [ docker ]
packages:
- apt-transport-https
- ca-certificates
- curl
- jq
- git
- gnupg
- lsb-release
- unattended-upgrades
runcmd:
- mkdir -p /etc/apt/keyrings
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
- echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
- apt-get update
- apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
- systemctl enable docker
- systemctl start docker
- curl -L -o /usr/local/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
- chmod +x /usr/local/bin/laconic-so