log diagnostics & errors to stderr

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -1,47 +0,0 @@
-name: Fixturenet-Eth-Plugeth-Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
-
-jobs:
-  test:
-    name: "Run an Ethereum plugeth fixturenet test"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth-plugeth/run-test.sh

.gitea/workflows/fixturenet-eth-test.yml

@@ -2,15 +2,7 @@ name: Fixturenet-Eth-Test
 
 on:
   push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
+    branches: 'ci-test'
 
 jobs:
   test:
@@ -19,16 +11,8 @@ jobs:
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
+      - name: "Install Python"
+        uses: cerc-io/setup-python@v4
         with:
           python-version: '3.8'
       - name: "Print Python version"
@@ -39,10 +23,5 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-eth tests"
         run: ./tests/fixturenet-eth/run-test.sh
-

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -1,55 +0,0 @@
-name: Fixturenet-Laconicd-Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
-
-jobs:
-  test:
-    name: "Run an Laconicd fixturenet test"
-    runs-on: ubuntu-latest
-    steps:
-      - name: 'Update'
-        run: apt-get update
-      - name: 'Setup jq'
-        run: apt-get install jq -y
-      - name: 'Check jq'
-        run: |
-          which jq
-          jq --version
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Run fixturenet-laconicd tests"
-        run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/publish.yml

@@ -5,8 +5,6 @@ on:
     branches:
       - main
       - publish-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
 
 jobs:
   publish:
@@ -20,16 +18,8 @@ jobs:
         run: |
           build_tag=$(./scripts/create_build_tag_file.sh)
           echo "build-tag=v${build_tag}" >> $GITHUB_OUTPUT
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
+      - name: "Install Python"
+        uses: cerc-io/setup-python@v4
         with:
           python-version: '3.8'
       - name: "Print Python version"
@@ -47,7 +37,7 @@ jobs:
         run: |
           cp ${{ steps.build.outputs.package-file }} ./laconic-so
       - name: "Create release"
-        uses: https://gitea.com/cerc-io/action-gh-release@gitea-v2
+        uses: cerc-io/action-gh-release@gitea-v1
         with:
           tag_name: ${{ steps.build-info.outputs.build-tag }}
           # On the publish test branch, mark our release as a draft

.gitea/workflows/test-deploy.yml

@@ -7,8 +7,6 @@ on:
     branches:
       - main
       - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
 
 # Needed until we can incorporate docker startup into the executor container
 env:
@@ -21,16 +19,8 @@ jobs:
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
+      - name: "Install Python"
+        uses: cerc-io/setup-python@v4
         with:
           python-version: '3.8'
       - name: "Print Python version"

.gitea/workflows/test-k8s-deploy.yml

@@ -1,55 +0,0 @@
-name: K8s Deploy Test
-
-on:
-  pull_request:
-    branches: '*'
-  push:
-    branches:
-      - main
-      - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
-jobs:
-  test:
-    name: "Run deploy test suite"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Install Go"
-        uses: actions/setup-go@v4
-        with:
-          go-version: '1.21'
-      - name: "Install Kind"
-        run: go install sigs.k8s.io/kind@v0.20.0
-      - name: "Debug Kind"
-        run: kind create cluster --retain && docker logs kind-control-plane

.gitea/workflows/test-webapp.yml

@@ -1,49 +0,0 @@
-name: Webapp Test
-
-on:
-  pull_request:
-    branches: '*'
-  push:
-    branches:
-      - main
-      - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
-jobs:
-  test:
-    name: "Run webapp test suite"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Run webapp tests"
-        run: ./tests/webapp-test/run-webapp-test.sh

.gitea/workflows/test.yml

@@ -7,8 +7,6 @@ on:
     branches:
       - main
       - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
 
 # Needed until we can incorporate docker startup into the executor container
 env:
@@ -21,16 +19,8 @@ jobs:
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
+      - name: "Install Python"
+        uses: cerc-io/setup-python@v4
         with:
           python-version: '3.8'
       - name: "Print Python version"
@@ -47,5 +37,3 @@ jobs:
           sleep 5
       - name: "Run smoke tests"
         run: ./tests/smoke-test/run-smoke-test.sh
-
-

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-plugeth-test CI job
-trigger

.gitea/workflows/triggers/fixturenet-eth-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-test CI job
-

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-laconicd-test CI job
-

.github/workflows/fixturenet-eth.yml

@@ -1,30 +0,0 @@
-name: Fixturenet-Eth Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.github/workflows/triggers/fixturenet-eth-test'
-
-jobs:
-  test:
-    name: "Run fixturenet-eth test suite"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      - name: "Install Python"
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth/run-test.sh

.github/workflows/fixturenet-laconicd.yml

@@ -1,30 +0,0 @@
-name: Fixturenet-Laconicd Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.github/workflows/triggers/fixturenet-laconicd-test'
-
-jobs:
-  test:
-    name: "Run fixturenet-laconicd test suite"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      - name: "Install Python"
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run fixturenet-laconicd tests"
-        run: ./tests/fixturenet-laconicd/run-test.sh

.github/workflows/lint.yml

@@ -1,21 +0,0 @@
-name: Lint Checks
-
-on:
-  pull_request:
-    branches: '*'
-  push:
-    branches: '*'
-
-jobs:
-  test:
-    name: "Run linter"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      - name: "Install Python"
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name : "Run flake8"
-        uses: py-actions/flake8@v2

.github/workflows/test-webapp.yml

@@ -1,29 +0,0 @@
-name: Webapp Test
-
-on:
-  pull_request:
-    branches: '*'
-  push:
-    branches: '*'
-
-jobs:
-  test:
-    name: "Run webapp test suite"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      - name: "Install Python"
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run webapp tests"
-        run: ./tests/webapp-test/run-webapp-test.sh

.github/workflows/triggers/fixturenet-eth-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-test CI job
-

.github/workflows/triggers/fixturenet-laconicd-test

@@ -1,3 +0,0 @@
-Change this file to trigger running the fixturenet-laconicd-test CI job
-
-trigger

.gitignore

@@ -6,5 +6,5 @@ laconic_stack_orchestrator.egg-info
 __pycache__
 *~
 package
-stack_orchestrator/data/build_tag.txt
-/build
+app/data/build_tag.txt
+build

README.md

@@ -6,7 +6,7 @@ Stack Orchestrator allows building and deployment of a Laconic Stack on a single
 
 ## Install
 
-**To get started quickly** on a fresh Ubuntu instance (e.g, Digital Ocean); [try this script](./scripts/quick-install-linux.sh). **WARNING:** always review scripts prior to running them so that you know what is happening on your machine.
+**To get started quickly** on a fresh Ubuntu instance (e.g, Digital Ocean); [try this script](./scripts/quick-install-ubuntu.sh). **WARNING:** always review scripts prior to running them so that you know what is happening on your machine.
 
 For any other installation, follow along below and **adapt these instructions based on the specifics of your system.**
 
@@ -16,7 +16,6 @@ Ensure that the following are already installed:
 - [Python3](https://wiki.python.org/moin/BeginnersGuide/Download): `python3 --version` >= `3.8.10` (the Python3 shipped in Ubuntu 20+ is good to go)
 - [Docker](https://docs.docker.com/get-docker/): `docker --version` >= `20.10.21`
 - [jq](https://stedolan.github.io/jq/download/): `jq --version` >= `1.5`
-- [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git): `git --version` >= `2.10.3`
 
 Note: if installing docker-compose via package manager on Linux (as opposed to Docker Desktop), you must [install the plugin](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually), e.g. :
 
@@ -49,27 +48,15 @@ Verify operation (your version will probably be different, just check here that
 laconic-so version
 Version: 1.1.0-7a607c2-202304260513
 ```
-Save the distribution url to `~/.laconic-so/config.yml`:
-```bash
-mkdir ~/.laconic-so
-echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" >  ~/.laconic-so/config.yml
-```
-
-### Update
-If Stack Orchestrator was installed using the process described above, it is able to subsequently self-update to the current latest version by running:
-
-```bash
-laconic-so update
-```
 
 ## Usage
 
-The various [stacks](/stack_orchestrator/data/stacks) each contain instructions for running different stacks based on your use case. For example:
+The various [stacks](/app/data/stacks) each contain instructions for running different stacks based on your use case. For example:
 
-- [self-hosted Gitea](/stack_orchestrator/data/stacks/build-support)
-- [an Optimism Fixturenet](/stack_orchestrator/data/stacks/fixturenet-optimism)
-- [laconicd with console and CLI](stack_orchestrator/data/stacks/fixturenet-laconic-loaded)
-- [kubo (IPFS)](stack_orchestrator/data/stacks/kubo)
+- [self-hosted Gitea](/app/data/stacks/build-support)
+- [an Optimism Fixturenet](/app/data/stacks/fixturenet-optimism)
+- [laconicd with console and CLI](app/data/stacks/fixturenet-laconic-loaded)
+- [kubo (IPFS)](app/data/stacks/kubo)
 
 ## Contributing
 

app/base.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Vulcanize
+# Copyright © 2022, 2023 Cerc
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,8 +15,9 @@
 
 import os
 from abc import ABC, abstractmethod
-from stack_orchestrator.deploy.deploy import get_stack_status
 from decouple import config
+from .deploy import get_stack_status
+from .util import _log
 
 
 def get_stack(config, stack):
@@ -49,7 +50,7 @@ class package_registry_stack(base_stack):
         url_from_environment = os.environ.get("CERC_NPM_REGISTRY_URL")
         if url_from_environment:
             if self.config.verbose:
-                print(f"Using package registry url from CERC_NPM_REGISTRY_URL: {url_from_environment}")
+                _log(f"Using package registry url from CERC_NPM_REGISTRY_URL: {url_from_environment}")
             self.url = url_from_environment
         else:
             # Otherwise we expect to use the local package-registry stack
@@ -58,14 +59,13 @@ class package_registry_stack(base_stack):
             if registry_running:
                 # If it is available, get its mapped port and construct its URL
                 if self.config.debug:
-                    print("Found local package registry stack is up")
+                    _log("Found local package registry stack is up")
                 # TODO: get url from deploy-stack
                 self.url = "http://gitea.local:3000/api/packages/cerc-io/npm/"
             else:
                 # If not, print a message about how to start it and return fail to the caller
-                print("ERROR: The package-registry stack is not running, and no external registry "
-                      "specified with CERC_NPM_REGISTRY_URL")
-                print("ERROR: Start the local package registry with: laconic-so --stack package-registry deploy-system up")
+                _log("ERROR: The package-registry stack is not running, and no external registry specified with CERC_NPM_REGISTRY_URL")
+                _log("ERROR: Start the local package registry with: laconic-so --stack package-registry deploy-system up")
                 return False
         return True
 
@@ -76,7 +76,5 @@ class package_registry_stack(base_stack):
 def get_npm_registry_url():
     # If an auth token is not defined, we assume the default should be the cerc registry
     # If an auth token is defined, we assume the local gitea should be used.
-    default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config(
-        "CERC_NPM_AUTH_TOKEN", default=None
-        ) else "https://git.vdb.to/api/packages/cerc-io/npm/"
+    default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config("CERC_NPM_AUTH_TOKEN", default=None) else "https://git.vdb.to/api/packages/cerc-io/npm/"
     return config("CERC_NPM_REGISTRY_URL", default=default_npm_registry_url)

app/build_containers.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Vulcanize
+# Copyright © 2022, 2023 Cerc
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -27,79 +27,12 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
-from stack_orchestrator.base import get_npm_registry_url
+from .util import _log, include_exclude_check, get_parsed_stack_config
+from .base import get_npm_registry_url
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
 
-def make_container_build_env(dev_root_path: str,
-                             container_build_dir: str,
-                             debug: bool,
-                             force_rebuild: bool,
-                             extra_build_args: str):
-    container_build_env = {
-        "CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
-        "CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""),
-        "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""),
-        "CERC_REPO_BASE_DIR": dev_root_path,
-        "CERC_CONTAINER_BASE_DIR": container_build_dir,
-        "CERC_HOST_UID": f"{os.getuid()}",
-        "CERC_HOST_GID": f"{os.getgid()}",
-        "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0")
-    }
-    container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
-    container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
-    container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {})
-    docker_host_env = os.getenv("DOCKER_HOST")
-    if docker_host_env:
-        container_build_env.update({"DOCKER_HOST": docker_host_env})
-
-    return container_build_env
-
-
-def process_container(container,
-                      container_build_dir: str,
-                      container_build_env: dict,
-                      dev_root_path: str,
-                      quiet: bool,
-                      verbose: bool,
-                      dry_run: bool,
-                      continue_on_error: bool,
-                      ):
-    if not quiet:
-        print(f"Building: {container}")
-    build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
-    build_script_filename = os.path.join(build_dir, "build.sh")
-    if verbose:
-        print(f"Build script filename: {build_script_filename}")
-    if os.path.exists(build_script_filename):
-        build_command = build_script_filename
-    else:
-        if verbose:
-            print(f"No script file found: {build_script_filename}, using default build script")
-        repo_dir = container.split('/')[1]
-        # TODO: make this less of a hack -- should be specified in some metadata somewhere
-        # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
-        repo_full_path = os.path.join(dev_root_path, repo_dir)
-        repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
-        build_command = os.path.join(container_build_dir,
-                                     "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
-    if not dry_run:
-        if verbose:
-            print(f"Executing: {build_command} with environment: {container_build_env}")
-        build_result = subprocess.run(build_command, shell=True, env=container_build_env)
-        if verbose:
-            print(f"Return code is: {build_result.returncode}")
-        if build_result.returncode != 0:
-            print(f"Error running build for {container}")
-            if not continue_on_error:
-                print("FATAL Error: container build failed and --continue-on-error not set, exiting")
-                sys.exit(1)
-            else:
-                print("****** Container Build Error, continuing because --continue-on-error is set")
-    else:
-        print("Skipped")
 
 @click.command()
 @click.option('--include', help="only build these containers")
@@ -119,22 +52,22 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-    container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
+    container_build_dir = Path(__file__).absolute().parent.joinpath("data", "container-build")
 
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
-        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
+        _log(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
     else:
         dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
     if not quiet:
-        print(f'Dev Root is: {dev_root_path}')
+        _log(f'Dev Root is: {dev_root_path}')
 
     if not os.path.isdir(dev_root_path):
-        print('Dev root directory doesn\'t exist, creating')
+        _log('Dev root directory doesn\'t exist, creating')
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from stack_orchestrator import data
+    from . import data
     with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
         all_containers = container_list_file.read().splitlines()
 
@@ -146,20 +79,64 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         containers_in_scope = all_containers
 
     if verbose:
-        print(f'Containers: {containers_in_scope}')
+        _log(f'Containers: {containers_in_scope}')
         if stack:
-            print(f"Stack: {stack}")
+            _log(f"Stack: {stack}")
 
-    container_build_env = make_container_build_env(dev_root_path,
-                                                   container_build_dir,
-                                                   debug,
-                                                   force_rebuild,
-                                                   extra_build_args)
+    # TODO: make this configurable
+    container_build_env = {
+        "CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
+        "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""),
+        "CERC_REPO_BASE_DIR": dev_root_path,
+        "CERC_CONTAINER_BASE_DIR": container_build_dir,
+        "CERC_HOST_UID": f"{os.getuid()}",
+        "CERC_HOST_GID": f"{os.getgid()}",
+        "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0")
+    }
+    container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
+    container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
+    container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {})
+    docker_host_env = os.getenv("DOCKER_HOST")
+    if docker_host_env:
+        container_build_env.update({"DOCKER_HOST": docker_host_env})
+
+    def process_container(container):
+        if not quiet:
+            _log(f"Building: {container}")
+        build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
+        build_script_filename = os.path.join(build_dir, "build.sh")
+        if verbose:
+            _log(f"Build script filename: {build_script_filename}")
+        if os.path.exists(build_script_filename):
+            build_command = build_script_filename
+        else:
+            if verbose:
+                _log(f"No script file found: {build_script_filename}, using default build script")
+            repo_dir = container.split('/')[1]
+            # TODO: make this less of a hack -- should be specified in some metadata somewhere
+            # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
+            repo_full_path = os.path.join(dev_root_path, repo_dir)
+            repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
+            build_command = os.path.join(container_build_dir, "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
+        if not dry_run:
+            if verbose:
+                _log(f"Executing: {build_command} with environment: {container_build_env}")
+            build_result = subprocess.run(build_command, shell=True, env=container_build_env)
+            if verbose:
+                _log(f"Return code is: {build_result.returncode}")
+            if build_result.returncode != 0:
+                _log(f"Error running build for {container}")
+                if not continue_on_error:
+                    _log("FATAL Error: container build failed and --continue-on-error not set, exiting")
+                    sys.exit(1)
+                else:
+                    _log("****** Container Build Error, continuing because --continue-on-error is set")
+        else:
+            _log("Skipped")
 
     for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
-            process_container(container, container_build_dir, container_build_env,
-                              dev_root_path, quiet, verbose, dry_run, continue_on_error)
+            process_container(container)
         else:
             if verbose:
-                print(f"Excluding: {container}")
+                _log(f"Excluding: {container}")

app/build_npms.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Vulcanize
+# Copyright © 2022, 2023 Cerc
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -25,17 +25,15 @@ from decouple import config
 import click
 import importlib.resources
 from python_on_whales import docker, DockerException
-from stack_orchestrator.base import get_stack
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
+from .base import get_stack
+from .util import include_exclude_check, get_parsed_stack_config
 
 builder_js_image_name = "cerc/builder-js:local"
 
-
 @click.command()
 @click.option('--include', help="only build these packages")
 @click.option('--exclude', help="don\'t build these packages")
-@click.option("--force-rebuild", is_flag=True, default=False,
-              help="Override existing target package version check -- force rebuild")
+@click.option("--force-rebuild", is_flag=True, default=False, help="Override existing target package version check -- force rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
 @click.pass_context
 def command(ctx, include, exclude, force_rebuild, extra_build_args):
@@ -83,7 +81,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         os.makedirs(build_root_path)
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from stack_orchestrator import data
+    from . import data
     with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file:
         all_packages = package_list_file.read().splitlines()
 
@@ -124,7 +122,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
             # envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             # but that isn't available in Python 3.8 (default in Ubuntu 20) so for now we use dict.update:
             envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token,
-                    "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml"  # Convention used by our web app packages
+                    "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml" # Convention used by our web app packages
                     }
             envs.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             envs.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})

app/data/compose/docker-compose-fixturenet-eth.yml

@@ -21,7 +21,7 @@ services:
     cap_add:
       - SYS_PTRACE
     environment:
-      CERC_REMOTE_DEBUG: ${CERC_REMOTE_DEBUG:-true}
+      CERC_REMOTE_DEBUG: "true"
       CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect}
       CERC_STATEDIFF_DB_NODE_ID: 1
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
@@ -40,7 +40,6 @@ services:
       - fixturenet-eth-bootnode-geth
     ports:
       - "8545"
-      - "8546"
       - "40000"
       - "6060"
 
@@ -62,9 +61,6 @@ services:
       - fixturenet-eth-bootnode-geth
     volumes:
       - fixturenet_eth_geth_2_data:/root/ethdata
-    ports:
-      - "8545"
-      - "8546"
 
   fixturenet-eth-bootnode-lighthouse:
     restart: always

app/data/compose/docker-compose-fixturenet-laconic-console.yml

@@ -3,7 +3,6 @@ services:
     restart: unless-stopped
     image: cerc/laconic-console-host:local
     environment:
-      - CERC_WEBAPP_FILES_DIR=${CERC_WEBAPP_FILES_DIR:-/usr/local/share/.config/yarn/global/node_modules/@cerc-io/console-app/dist/production}
       - LACONIC_HOSTED_ENDPOINT=${LACONIC_HOSTED_ENDPOINT:-http://localhost}
     ports:
       - "80"

app/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -15,7 +15,7 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473"
+      - "9473:9473"
       - "8545"
       - "8546"
       - "9090"

app/data/compose/docker-compose-fixturenet-lotus.yml

@@ -8,10 +8,17 @@ services:
     image: cerc/lotus:local
     volumes:
       - ../config/fixturenet-lotus/setup-miner.sh:/docker-entrypoint-scripts.d/setup-miner.sh
-      - ../config/fixturenet-lotus/fund-account.sh:/fund-account.sh
-      - lotus_miner_params:/var/tmp/filecoin-proof-parameters
-      - lotus_shared:/root/.lotus-shared
-      - lotus_miner_data:/root/data
+      - ../config/fixturenet-lotus/genesis/devgen.car:/devgen.car
+      - $HOME/stack-orchestrator/app/data/config/fixturenet-lotus/genesis/.genesis-sectors:/root/.genesis-sectors
+      - lotus-shared:/root/.lotus-shared
+    healthcheck:
+#      test: ["CMD-SHELL", "grep 'started ChainNotify channel' /var/log/lotus.log"]
+#      test: ["CMD-SHELL", "[ -f /root/.lotus-shared/miner.addr ]"]
+      test: ["CMD-SHELL", "[ -d /root/.lotus-miner-local-net ]"]
+      interval: 10s
+      timeout: 10s
+      retries: 10
+      start_period: 60s
     entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-miner.sh"]
     ports:
       - "1234"
@@ -23,23 +30,14 @@ services:
     hostname: lotus-node-1
     env_file:
       - ../config/fixturenet-lotus/lotus-env.env
-    environment:
-      # Use 0.0.0.0 so that calls can be made from outside the container
-      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
     image: cerc/lotus:local
     volumes:
       - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
-      - lotus_node_1_params:/var/tmp/filecoin-proof-parameters
-      - lotus_shared:/root/.lotus-shared
-      - lotus_node_1_data:/root/data
-    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "1234"]
-      interval: 30s
-      timeout: 10s
-      retries: 60
-      start_period: 3s
+      - ../config/fixturenet-lotus/genesis/devgen.car:/devgen.car
+      - lotus-shared:/root/.lotus-shared
     depends_on:
-      - lotus-miner
+      lotus-miner:
+        condition: service_healthy
     entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
     ports:
       - "1234"
@@ -51,23 +49,14 @@ services:
     hostname: lotus-node-2
     env_file:
       - ../config/fixturenet-lotus/lotus-env.env
-    environment:
-      # Use 0.0.0.0 so that calls can be made from outside the container
-      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
     image: cerc/lotus:local
     volumes:
       - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
-      - lotus_node_2_params:/var/tmp/filecoin-proof-parameters
-      - lotus_shared:/root/.lotus-shared
-      - lotus_node_2_data:/root/data
-    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "1234"]
-      interval: 30s
-      timeout: 10s
-      retries: 60
-      start_period: 3s
+      - ../config/fixturenet-lotus/genesis/devgen.car:/devgen.car
+      - lotus-shared:/root/.lotus-shared
     depends_on:
-      - lotus-miner
+      lotus-miner:
+        condition: service_healthy
     entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
     ports:
       - "1234"
@@ -76,10 +65,4 @@ services:
       - "1777"
 
 volumes:
-  lotus_miner_params:
-  lotus_node_1_params:
-  lotus_node_2_params:
-  lotus_shared:
-  lotus_miner_data:
-  lotus_node_1_data:
-  lotus_node_2_data:
+  lotus-shared:

app/data/compose/docker-compose-fixturenet-optimism.yml

@@ -23,10 +23,10 @@ services:
     command: |
       "./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./run.sh"
     volumes:
-      - ../config/network/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh
-      - ../config/optimism-contracts/hardhat-tasks/verify-contract-deployment.ts:/app/packages/contracts-bedrock/tasks/verify-contract-deployment.ts
-      - ../config/optimism-contracts/hardhat-tasks/rekey-json.ts:/app/packages/contracts-bedrock/tasks/rekey-json.ts
-      - ../config/optimism-contracts/hardhat-tasks/send-balance.ts:/app/packages/contracts-bedrock/tasks/send-balance.ts
+      - ../config/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh
+      - ../container-build/cerc-optimism-contracts/hardhat-tasks/verify-contract-deployment.ts:/app/packages/contracts-bedrock/tasks/verify-contract-deployment.ts
+      - ../container-build/cerc-optimism-contracts/hardhat-tasks/rekey-json.ts:/app/packages/contracts-bedrock/tasks/rekey-json.ts
+      - ../container-build/cerc-optimism-contracts/hardhat-tasks/send-balance.ts:/app/packages/contracts-bedrock/tasks/send-balance.ts
       - ../config/fixturenet-optimism/optimism-contracts/update-config.js:/app/packages/contracts-bedrock/update-config.js
       - ../config/fixturenet-optimism/optimism-contracts/run.sh:/app/packages/contracts-bedrock/run.sh
       - l2_accounts:/l2-accounts
@@ -70,7 +70,6 @@ services:
     command: "/run-op-geth.sh"
     ports:
       - "0.0.0.0:8545:8545"
-      - "0.0.0.0:8546:8546"
     healthcheck:
       test: ["CMD", "nc", "-vz", "localhost:8545"]
       interval: 30s
@@ -121,7 +120,7 @@ services:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_L1_RPC: ${CERC_L1_RPC}
     volumes:
-      - ../config/network/wait-for-it.sh:/wait-for-it.sh
+      - ../config/wait-for-it.sh:/wait-for-it.sh
       - ../config/fixturenet-optimism/run-op-batcher.sh:/run-op-batcher.sh
       - l2_accounts:/l2-accounts:ro
     entrypoint: ["sh", "-c"]
@@ -146,7 +145,7 @@ services:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_L1_RPC: ${CERC_L1_RPC}
     volumes:
-      - ../config/network/wait-for-it.sh:/wait-for-it.sh
+      - ../config/wait-for-it.sh:/wait-for-it.sh
       - ../config/fixturenet-optimism/run-op-proposer.sh:/run-op-proposer.sh
       - l1_deployment:/contracts-bedrock:ro
       - l2_accounts:/l2-accounts:ro

app/data/compose/docker-compose-fixturenet-plugeth.yml

@@ -9,6 +9,7 @@ services:
     image: cerc/fixturenet-plugeth-plugeth:local
     volumes:
       - fixturenet_plugeth_bootnode_geth_data:/root/ethdata
+      - ../config/fixturenet-plugeth/plugins:/root/ethdata/plugins
     ports:
       - "9898"
       - "30303"
@@ -19,8 +20,8 @@ services:
     cap_add:
       - SYS_PTRACE
     environment:
-      CERC_REMOTE_DEBUG: ${CERC_REMOTE_DEBUG:-true}
-      CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect}
+      CERC_REMOTE_DEBUG: "true"
+      CERC_RUN_STATEDIFF: "detect"
       CERC_STATEDIFF_DB_NODE_ID: 1
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     env_file:
@@ -38,7 +39,6 @@ services:
       - fixturenet-eth-bootnode-geth
     ports:
       - "8545"
-      - "8546"
       - "40000"
       - "6060"
 
@@ -60,16 +60,13 @@ services:
       - fixturenet-eth-bootnode-geth
     volumes:
       - fixturenet_plugeth_geth_2_data:/root/ethdata
-    ports:
-      - "8545"
-      - "8546"
 
   fixturenet-eth-bootnode-lighthouse:
     restart: always
     hostname: fixturenet-eth-bootnode-lighthouse
     environment:
       RUN_BOOTNODE: "true"
-    image: cerc/fixturenet-eth-lighthouse:local
+    image: cerc/fixturenet-plugeth-lighthouse:local
 
   fixturenet-eth-lighthouse-1:
     restart: always
@@ -86,7 +83,7 @@ services:
       NODE_NUMBER: "1"
       ETH1_ENDPOINT: "http://fixturenet-eth-geth-1:8545"
       EXECUTION_ENDPOINT: "http://fixturenet-eth-geth-1:8551"
-    image: cerc/fixturenet-eth-lighthouse:local
+    image: cerc/fixturenet-plugeth-lighthouse:local
     volumes:
       - fixturenet_plugeth_lighthouse_1_data:/opt/testnet/build/cl
     depends_on:
@@ -113,7 +110,7 @@ services:
       ETH1_ENDPOINT: "http://fixturenet-eth-geth-2:8545"
       EXECUTION_ENDPOINT: "http://fixturenet-eth-geth-2:8551"
       LIGHTHOUSE_GENESIS_STATE_URL: "http://fixturenet-eth-lighthouse-1:8001/eth/v2/debug/beacon/states/0"
-    image: cerc/fixturenet-eth-lighthouse:local
+    image: cerc/fixturenet-plugeth-lighthouse:local
     volumes:
       - fixturenet_plugeth_lighthouse_2_data:/opt/testnet/build/cl
     depends_on:

app/data/compose/docker-compose-ipld-eth-server.yml

@@ -25,8 +25,8 @@ services:
       PROM_HTTP: "true"
       PROM_HTTP_ADDR: "0.0.0.0"
       PROM_HTTP_PORT: "8090"
-      LOG_LEVEL: "debug"
-      CERC_REMOTE_DEBUG: ${CERC_REMOTE_DEBUG:-true}
+      LOGRUS_LEVEL: "debug"
+      CERC_REMOTE_DEBUG: "true"
     volumes:
       - type: bind
         source: ../config/ipld-eth-server/chain.json

app/data/compose/docker-compose-kubo.yml

@@ -2,7 +2,7 @@ version: "3.2"
 # See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up
 services:
   ipfs:
-    image: ipfs/kubo:v0.24.0
+    image: ipfs/kubo:master-2023-02-20-714a968
     restart: always
     volumes:
       - ./ipfs/import:/import

app/data/compose/docker-compose-mainnet-laconicd.yml

@@ -2,22 +2,20 @@ services:
   laconicd:
     restart: no
     image: cerc/laconicd:local
-    command: ["/bin/sh", "-c", "/opt/run-laconicd.sh"]
+    command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
     volumes:
       # The cosmos-sdk node's database directory:
       - laconicd-data:/root/.laconicd/data
-      - laconicd-config:/root/.laconicd/config
-      - laconicd-keyring:/root/.laconicd/keyring-test
       # TODO: look at folding these scripts into the container
-      - ../config/mainnet-laconicd/scripts/run-laconicd.sh:/opt/run-laconicd.sh
-      - ../config/mainnet-laconicd/scripts/export-mykey.sh:/docker-entrypoint-scripts.d/export-mykey.sh
-      - ../config/mainnet-laconicd/scripts/export-myaddress.sh:/docker-entrypoint-scripts.d/export-myaddress.sh
+      - ../config/mainnet-laconicd/create-fixturenet.sh:/docker-entrypoint-scripts.d/create-fixturenet.sh
+      - ../config/mainnet-laconicd/export-mykey.sh:/docker-entrypoint-scripts.d/export-mykey.sh
+      - ../config/mainnet-laconicd/export-myaddress.sh:/docker-entrypoint-scripts.d/export-myaddress.sh
     # TODO: determine which of the ports below is really needed
     ports:
       - "6060"
       - "26657"
       - "26656"
-      - "9473"
+      - "9473:9473"
       - "8545"
       - "8546"
       - "9090"
@@ -30,5 +28,3 @@ services:
 
 volumes:
   laconicd-data:
-  laconicd-config:
-  laconicd-keyring:

app/data/compose/docker-compose-mobymask-app.yml

@@ -14,15 +14,12 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_PUBSUB: ${CERC_PUBSUB}
-      CERC_RELEASE: "v0.1.7"
-      CERC_USE_NPM: true
-      CERC_CONFIG_FILE: "src/config.json"
+      CERC_BUILD_DIR: "@cerc-io/mobymask-ui/build"
     working_dir: /scripts
     command: ["sh", "mobymask-app-start.sh"]
     volumes:
+      - ../config/wait-for-it.sh:/scripts/wait-for-it.sh
       - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
-      - ../config/watcher-mobymask-v2/mobymask-app-config.json:/app/src/mobymask-app-config.json
       - peers_ids:/peers
       - mobymask_deployment:/server
     ports:
@@ -49,15 +46,12 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_PUBSUB: ${CERC_PUBSUB}
-      CERC_RELEASE: "v0.1.7-lxdao-0.1.1"
-      CERC_USE_NPM: false
-      CERC_CONFIG_FILE: "src/utils/config.json"
+      CERC_BUILD_DIR: "@cerc-io/mobymask-ui-lxdao/build"
     working_dir: /scripts
     command: ["sh", "mobymask-app-start.sh"]
     volumes:
+      - ../config/wait-for-it.sh:/scripts/wait-for-it.sh
       - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
-      - ../config/watcher-mobymask-v2/mobymask-app-config.json:/app/src/mobymask-app-config.json
       - peers_ids:/peers
       - mobymask_deployment:/server
     ports:

app/data/compose/docker-compose-peer-test-app.yml

@@ -14,7 +14,7 @@ services:
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
     command: ["sh", "test-app-start.sh"]
     volumes:
-      - ../config/network/wait-for-it.sh:/scripts/wait-for-it.sh
+      - ../config/wait-for-it.sh:/scripts/wait-for-it.sh
       - ../config/watcher-mobymask-v2/test-app-start.sh:/scripts/test-app-start.sh
       - peers_ids:/peers
     ports:

app/data/compose/docker-compose-test.yml

@@ -4,7 +4,6 @@ services:
     restart: always
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED}
     volumes:
       - test-data:/data
     ports:

app/data/compose/docker-compose-watcher-erc20.yml

@@ -34,7 +34,7 @@ services:
       - ETH_RPC_URL=http://go-ethereum:8545
     command: ["sh", "-c", "yarn server"]
     volumes:
-      - ../config/watcher-erc20/erc20-watcher.toml:/app/environments/local.toml
+      - ../config/watcher-erc20/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml
     ports:
       - "0.0.0.0:3002:3001"
       - "0.0.0.0:9002:9001"

app/data/compose/docker-compose-watcher-mobymask-v2.yml

@@ -44,7 +44,7 @@ services:
       CERC_L2_NODE_PORT: ${CERC_L2_NODE_PORT}
     command: ["sh", "deploy-and-generate-invite.sh"]
     volumes:
-      - ../config/network/wait-for-it.sh:/app/packages/server/wait-for-it.sh
+      - ../config/wait-for-it.sh:/app/packages/server/wait-for-it.sh
       - ../config/watcher-mobymask-v2/secrets-template.json:/app/packages/server/secrets-template.json
       - ../config/watcher-mobymask-v2/deploy-and-generate-invite.sh:/app/packages/server/deploy-and-generate-invite.sh
       - mobymask_deployment:/app/packages/server
@@ -84,7 +84,6 @@ services:
       CERC_PRIVATE_KEY_PEER: ${CERC_PRIVATE_KEY_PEER}
       CERC_RELAY_PEERS: ${CERC_RELAY_PEERS}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_PUBSUB: ${CERC_PUBSUB}
       CERC_RELAY_ANNOUNCE_DOMAIN: ${CERC_RELAY_ANNOUNCE_DOMAIN}
       CERC_ENABLE_PEER_L2_TXS: ${CERC_ENABLE_PEER_L2_TXS}
       CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}

app/data/compose/docker-compose-watcher-mobymask.yml

@@ -24,37 +24,15 @@ services:
       retries: 15
       start_period: 10s
 
-  mobymask-watcher-job-runner:
-    restart: unless-stopped
-    depends_on:
-      mobymask-watcher-db:
-        condition: service_healthy
-    image: cerc/watcher-mobymask:local
-    command: ["sh", "-c", "yarn job-runner"]
-    volumes:
-      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/environments/local.toml
-    ports:
-      - "0.0.0.0:9000:9000"
-    extra_hosts:
-      - "ipld-eth-server:host-gateway"
-    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 5s
-
   mobymask-watcher-server:
     restart: unless-stopped
     depends_on:
       mobymask-watcher-db:
         condition: service_healthy
-      mobymask-watcher-job-runner:
-        condition: service_healthy
     image: cerc/watcher-mobymask:local
     command: ["sh", "-c", "yarn server"]
     volumes:
-      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/environments/local.toml
+      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml
     ports:
       - "0.0.0.0:3001:3001"
       - "0.0.0.0:9001:9001"
@@ -67,5 +45,21 @@ services:
       retries: 15
       start_period: 5s
 
+  mobymask-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      mobymask-watcher-server:
+        condition: service_healthy
+      mobymask-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-mobymask:local
+    command: ["sh", "-c", "yarn job-runner"]
+    volumes:
+      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml
+    ports:
+      - "0.0.0.0:9000:9000"
+    extra_hosts:
+      - "ipld-eth-server:host-gateway"
+
 volumes:
   mobymask_watcher_db_data:

app/data/config/fixturenet-eth/fixturenet-eth.env

@@ -18,8 +18,7 @@ CERC_STATEDIFF_DB_NAME="cerc_testing"
 CERC_STATEDIFF_DB_USER="vdbm"
 CERC_STATEDIFF_DB_PASSWORD="password"
 CERC_STATEDIFF_DB_GOOSE_MIN_VER=${CERC_STATEDIFF_DB_GOOSE_MIN_VER:-18}
-CERC_STATEDIFF_DB_LOG_STATEMENTS="${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false}"
+CERC_STATEDIFF_DB_LOG_STATEMENTS="false"
 CERC_STATEDIFF_WORKERS=2
 
 CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"
-CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}

app/data/config/fixturenet-lotus/genesis/.genesis-sectors/cache/s-t01000-0/p_aux

@@ -0,0 +1 @@
+}+V�{iνΆΠΉ�²�¨ΣΗ\k»qς␍—?δΪAΒfκ’~μ©™LΉ�tb
·yqτ·²ηξΔ�Ο?ξaΣ�J

app/data/config/fixturenet-lotus/genesis/.genesis-sectors/cache/s-t01000-0/sc-02-data-tree-r-last.dat

@@ -0,0 +1 @@
+Βfκ’~μ©™LΉ�tb
·yqτ·²ηξΔ�Ο?ξaΣ�J

app/data/config/fixturenet-lotus/genesis/.genesis-sectors/pre-seal-t01000.json

@@ -0,0 +1,71 @@
+{
+  "t01000": {
+    "ID": "t01000",
+    "Owner": "t3spusn5ia57qezc3fwpe3n2lhb4y4xt67xoflqbqy2muliparw2uktevletuv7gl4qakjpafgcl7jk2s2er3q",
+    "Worker": "t3spusn5ia57qezc3fwpe3n2lhb4y4xt67xoflqbqy2muliparw2uktevletuv7gl4qakjpafgcl7jk2s2er3q",
+    "PeerId": "12D3KooWG5q6pWJVdPBhDBv9AjWVbUh4xxTAZ7xvgZSjczWuD2Z9",
+    "MarketBalance": "0",
+    "PowerBalance": "0",
+    "SectorSize": 2048,
+    "Sectors": [
+      {
+        "CommR": {
+          "/": "bagboea4b5abcboxypcewlkmrat2myu4vthk3ii2pcomak7nhqmdbb6sxlolp2wdf"
+        },
+        "CommD": {
+          "/": "baga6ea4seaqn3jfixthmdgksv4vhfeuyvr6upw6tvaqbmzmsyxnzosm4pwgnmlq"
+        },
+        "SectorID": 0,
+        "Deal": {
+          "PieceCID": {
+            "/": "baga6ea4seaqn3jfixthmdgksv4vhfeuyvr6upw6tvaqbmzmsyxnzosm4pwgnmlq"
+          },
+          "PieceSize": 2048,
+          "VerifiedDeal": false,
+          "Client": "t3spusn5ia57qezc3fwpe3n2lhb4y4xt67xoflqbqy2muliparw2uktevletuv7gl4qakjpafgcl7jk2s2er3q",
+          "Provider": "t01000",
+          "Label": "0",
+          "StartEpoch": 0,
+          "EndEpoch": 9001,
+          "StoragePricePerEpoch": "0",
+          "ProviderCollateral": "0",
+          "ClientCollateral": "0"
+        },
+        "DealClientKey": {
+          "Type": "bls",
+          "PrivateKey": "tFvSRiSg2G3Ssgg0PSYy23XyjaIMXpsmdyG2B7UFLT4="
+        },
+        "ProofType": 5
+      },
+      {
+        "CommR": {
+          "/": "bagboea4b5abcb6krzypqcczhcnbeyjcqkeo6omfergm336o3kitugh3jgjog2yqq"
+        },
+        "CommD": {
+          "/": "baga6ea4seaqhondpb2373hjasjplxvbjzi5n5mm4fbbhjxp5ptnbq4cibapkeii"
+        },
+        "SectorID": 1,
+        "Deal": {
+          "PieceCID": {
+            "/": "baga6ea4seaqhondpb2373hjasjplxvbjzi5n5mm4fbbhjxp5ptnbq4cibapkeii"
+          },
+          "PieceSize": 2048,
+          "VerifiedDeal": false,
+          "Client": "t3spusn5ia57qezc3fwpe3n2lhb4y4xt67xoflqbqy2muliparw2uktevletuv7gl4qakjpafgcl7jk2s2er3q",
+          "Provider": "t01000",
+          "Label": "1",
+          "StartEpoch": 0,
+          "EndEpoch": 9001,
+          "StoragePricePerEpoch": "0",
+          "ProviderCollateral": "0",
+          "ClientCollateral": "0"
+        },
+        "DealClientKey": {
+          "Type": "bls",
+          "PrivateKey": "tFvSRiSg2G3Ssgg0PSYy23XyjaIMXpsmdyG2B7UFLT4="
+        },
+        "ProofType": 5
+      }
+    ]
+  }
+}

app/data/config/fixturenet-lotus/genesis/.genesis-sectors/pre-seal-t01000.key

@@ -0,0 +1 @@
+7b2254797065223a22626c73222c22507269766174654b6579223a227446765352695367324733537367673050535979323358796a61494d5870736d64794732423755464c54343d227d

app/data/config/fixturenet-lotus/genesis/.genesis-sectors/sectorstore.json

@@ -0,0 +1,11 @@
+{
+  "ID": "f355523e-69d0-4984-bd0e-9588487c6231",
+  "Weight": 0,
+  "CanSeal": false,
+  "CanStore": false,
+  "MaxStorage": 0,
+  "Groups": null,
+  "AllowTo": null,
+  "AllowTypes": null,
+  "DenyTypes": null
+}

app/data/config/fixturenet-lotus/genesis/localnet.json

@@ -0,0 +1,108 @@
+{
+  "NetworkVersion": 18,
+  "Accounts": [
+    {
+      "Type": "account",
+      "Balance": "50000000000000000000000000",
+      "Meta": {
+        "Owner": "t3spusn5ia57qezc3fwpe3n2lhb4y4xt67xoflqbqy2muliparw2uktevletuv7gl4qakjpafgcl7jk2s2er3q"
+      }
+    }
+  ],
+  "Miners": [
+    {
+      "ID": "t01000",
+      "Owner": "t3spusn5ia57qezc3fwpe3n2lhb4y4xt67xoflqbqy2muliparw2uktevletuv7gl4qakjpafgcl7jk2s2er3q",
+      "Worker": "t3spusn5ia57qezc3fwpe3n2lhb4y4xt67xoflqbqy2muliparw2uktevletuv7gl4qakjpafgcl7jk2s2er3q",
+      "PeerId": "12D3KooWG5q6pWJVdPBhDBv9AjWVbUh4xxTAZ7xvgZSjczWuD2Z9",
+      "MarketBalance": "0",
+      "PowerBalance": "0",
+      "SectorSize": 2048,
+      "Sectors": [
+        {
+          "CommR": {
+            "/": "bagboea4b5abcboxypcewlkmrat2myu4vthk3ii2pcomak7nhqmdbb6sxlolp2wdf"
+          },
+          "CommD": {
+            "/": "baga6ea4seaqn3jfixthmdgksv4vhfeuyvr6upw6tvaqbmzmsyxnzosm4pwgnmlq"
+          },
+          "SectorID": 0,
+          "Deal": {
+            "PieceCID": {
+              "/": "baga6ea4seaqn3jfixthmdgksv4vhfeuyvr6upw6tvaqbmzmsyxnzosm4pwgnmlq"
+            },
+            "PieceSize": 2048,
+            "VerifiedDeal": false,
+            "Client": "t3spusn5ia57qezc3fwpe3n2lhb4y4xt67xoflqbqy2muliparw2uktevletuv7gl4qakjpafgcl7jk2s2er3q",
+            "Provider": "t01000",
+            "Label": "0",
+            "StartEpoch": 0,
+            "EndEpoch": 9001,
+            "StoragePricePerEpoch": "0",
+            "ProviderCollateral": "0",
+            "ClientCollateral": "0"
+          },
+          "DealClientKey": {
+            "Type": "bls",
+            "PrivateKey": "tFvSRiSg2G3Ssgg0PSYy23XyjaIMXpsmdyG2B7UFLT4="
+          },
+          "ProofType": 5
+        },
+        {
+          "CommR": {
+            "/": "bagboea4b5abcb6krzypqcczhcnbeyjcqkeo6omfergm336o3kitugh3jgjog2yqq"
+          },
+          "CommD": {
+            "/": "baga6ea4seaqhondpb2373hjasjplxvbjzi5n5mm4fbbhjxp5ptnbq4cibapkeii"
+          },
+          "SectorID": 1,
+          "Deal": {
+            "PieceCID": {
+              "/": "baga6ea4seaqhondpb2373hjasjplxvbjzi5n5mm4fbbhjxp5ptnbq4cibapkeii"
+            },
+            "PieceSize": 2048,
+            "VerifiedDeal": false,
+            "Client": "t3spusn5ia57qezc3fwpe3n2lhb4y4xt67xoflqbqy2muliparw2uktevletuv7gl4qakjpafgcl7jk2s2er3q",
+            "Provider": "t01000",
+            "Label": "1",
+            "StartEpoch": 0,
+            "EndEpoch": 9001,
+            "StoragePricePerEpoch": "0",
+            "ProviderCollateral": "0",
+            "ClientCollateral": "0"
+          },
+          "DealClientKey": {
+            "Type": "bls",
+            "PrivateKey": "tFvSRiSg2G3Ssgg0PSYy23XyjaIMXpsmdyG2B7UFLT4="
+          },
+          "ProofType": 5
+        }
+      ]
+    }
+  ],
+  "NetworkName": "localnet-6d52dae5-ff29-4bac-a45d-f84e6c07564c",
+  "VerifregRootKey": {
+    "Type": "multisig",
+    "Balance": "0",
+    "Meta": {
+      "Signers": [
+        "t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy"
+      ],
+      "Threshold": 1,
+      "VestingDuration": 0,
+      "VestingStart": 0
+    }
+  },
+  "RemainderAccount": {
+    "Type": "multisig",
+    "Balance": "0",
+    "Meta": {
+      "Signers": [
+        "t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy"
+      ],
+      "Threshold": 1,
+      "VestingDuration": 0,
+      "VestingStart": 0
+    }
+  }
+}

app/data/config/fixturenet-lotus/lotus-env.env

@@ -0,0 +1,5 @@
+LOTUS_PATH=~/.lotus-local-net
+LOTUS_MINER_PATH=~/.lotus-miner-local-net
+LOTUS_SKIP_GENESIS_CHECK=_yes_
+CGO_CFLAGS_ALLOW="-D__BLST_PORTABLE__"
+CGO_CFLAGS="-D__BLST_PORTABLE__"

app/data/config/fixturenet-lotus/setup-miner.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+lotus --version
+
+# # remove old bootnode peer info if present
+# [ -f /root/.lotus-shared/miner.addr ] && rm /root/.lotus-shared/miner.addr
+
+##TODO: generate genesis files inside container instead of bundling in config dir
+##something like commands below should work, other scripts/compose will have to be updated to corresponding directories
+# lotus fetch-params 2048
+# lotus-seed pre-seal --sector-size 2KiB --num-sectors 2
+# lotus-seed genesis new localnet.json
+# lotus-seed genesis add-miner localnet.json ~/.genesis-sectors/pre-seal-t01000.json
+
+
+# start daemon
+nohup lotus daemon  --genesis=/devgen.car --profile=bootstrapper --bootstrap=false > /var/log/lotus.log 2>&1 &
+
+# Loop until the daemon is started
+echo "Waiting for daemon to start..."
+while ! grep -q "started ChainNotify channel" /var/log/lotus.log ; do
+    sleep 5
+done
+echo "Daemon started."
+
+# publish bootnode peer info to shared volume
+lotus net listen | awk 'NR==1{print}' > /root/.lotus-shared/miner.addr
+
+# if miner not already initialized
+if [ ! -d /root/.lotus-miner-local-net ]; then
+  # initialize miner
+  lotus wallet import --as-default ~/.genesis-sectors/pre-seal-t01000.key
+  lotus-miner init --genesis-miner --actor=t01000 --sector-size=2KiB --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json --nosync
+fi
+
+# start miner
+nohup lotus-miner run --nosync &
+
+tail -f /dev/null

app/data/config/fixturenet-lotus/setup-node.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+lotus --version
+
+##TODO: paths can use values from lotus-env.env file
+
+# if not already initialized
+if [ ! -f /root/.lotus-local-net/config.toml ]; then
+  # init node config
+  mkdir $HOME/.lotus-local-net
+  lotus config default > $HOME/.lotus-local-net/config.toml
+
+  # add bootstrap peer info if available
+  if [ -f /root/.lotus-shared/miner.addr ]; then
+    MINER_ADDR=\"$(cat /root/.lotus-shared/miner.addr)\"
+    # add bootstrap peer id to config file
+    sed -i "/^\[Libp2p\]/a \ \ BootstrapPeers = [$MINER_ADDR]" $HOME/.lotus-local-net/config.toml
+  else
+    echo "Bootstrap peer info not found, unable to configure. Manual peering will be required."
+  fi
+fi
+
+# start node
+lotus daemon --genesis=/devgen.car

app/data/config/fixturenet-plugeth/plugins/README.md

@@ -0,0 +1 @@
+See: https://docs.plugeth.org/