Merge pull request #108 from cerc-io/telackey/path

fix: import Path

README.md

@@ -3,9 +3,7 @@
 Stack Orchestrator allows building and deployment of a Laconic stack on a single machine with minimial prerequisites.
 
 ## Setup
-### User Mode
-User mode runs the orchestrator from a "binary" single-file release and does not require special Python environment setup. Use this mode unless you plan to make changes to the orchestrator source code.
-#### Prerequisites
+### Prerequisites
 Stack Orchestrator is a Python3 CLI tool that runs on any OS with Python3 and Docker. Tested on: Ubuntu 20/22.
 
 Ensure that the following are already installed:
@@ -30,11 +28,17 @@ Ensure that the following are already installed:
    # see https://docs.docker.com/compose/install/linux/#install-the-plugin-manually for further details
    # or to install for all users.
    ```
-#### Install
+
+### User Mode Install
+
+User mode runs the orchestrator from a "binary" single-file release and does not require special Python environment setup. Use this mode unless you plan to make changes to the orchestrator source code.
+
+*NOTE: User Mode is currently broken, use "Developer mode" described below for now.*
+
 1. Download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags), into a suitable directory (e.g. `~/bin`):
    ```
    $ cd ~/bin
-   $ curl https://github.com/cerc-io/stack-orchestrator/releases/download/v1.0.1-alpha/laconic-so
+   $ curl -L https://github.com/cerc-io/stack-orchestrator/releases/download/v1.0.3-alpha/laconic-so
    ```
 1. Ensure `laconic-so` is on the `PATH`
 1. Verify operation:
@@ -56,7 +60,7 @@ Ensure that the following are already installed:
      deploy-system       deploy a stack
      setup-repositories  git clone the set of repositories required to build...
    ```
-### Developer mode
+### Developer mode Install
 Suitable for developers either modifying or debugging the orchestrator Python code:
 #### Prerequisites
 In addition to the binary install prerequisites listed above, the following are required:

app/build_containers.py

@@ -21,11 +21,11 @@
 # TODO: display the available list of containers; allow re-build of either all or specific containers
 
 import os
-import sys
 from decouple import config
 import subprocess
 import click
-import pkg_resources
+import importlib
+from pathlib import Path
 from .util import include_exclude_check
 
 # TODO: find a place for this
@@ -44,6 +44,9 @@ def command(ctx, include, exclude):
     dry_run = ctx.obj.dry_run
     local_stack = ctx.obj.local_stack
 
+    # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
+    container_build_dir = Path(__file__).absolute().parent.joinpath("data", "container-build")
+
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
         print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
@@ -56,24 +59,43 @@ def command(ctx, include, exclude):
     if not os.path.isdir(dev_root_path):
         print('Dev root directory doesn\'t exist, creating')
 
-    with pkg_resources.resource_stream(__name__, "data/container-image-list.txt") as container_list_file:
-        containers = container_list_file.read().decode().splitlines()
+    # See: https://stackoverflow.com/a/20885799/1701505
+    from . import data
+    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+        containers = container_list_file.read().splitlines()
 
     if verbose:
         print(f'Containers: {containers}')
 
+    # TODO: make this configurable
+    container_build_env = {
+        "CERC_NPM_URL": "http://gitea.local:3000/api/packages/cerc-io/npm/",
+        "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default="<token-not-supplied>"),
+        "CERC_REPO_BASE_DIR": dev_root_path
+    }
+
     def process_container(container):
         if not quiet:
             print(f"Building: {container}")
-        build_script_filename = os.path.join("container-build", container.replace("/", "-"), "build.sh")
+        build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
+        build_script_filename = os.path.join(build_dir, "build.sh")
         if verbose:
-            print(f"Script: {build_script_filename}")
-        if not os.path.exists(build_script_filename):
-            print(f"Error, script: {build_script_filename} doesn't exist")
-            sys.exit(1)
+            print(f"Build script filename: {build_script_filename}")
+        if os.path.exists(build_script_filename):
+            build_command = build_script_filename
+        else:
+            if verbose:
+                print(f"No script file found: {build_script_filename}, using default build script")
+            repo_dir = container.split('/')[1]
+            # TODO: make this less of a hack -- should be specified in some metadata somewhere
+            # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
+            repo_full_path = os.path.join(dev_root_path, repo_dir)
+            repo_dir_or_build_dir = repo_dir if os.path.exists(repo_full_path) else build_dir
+            build_command = os.path.join(container_build_dir, "default-build.sh") + f" {container} {repo_dir_or_build_dir}"
         if not dry_run:
-            # We need to export CERC_REPO_BASE_DIR
-            build_result = subprocess.run(build_script_filename, shell=True, env={'CERC_REPO_BASE_DIR': dev_root_path})
+            if verbose:
+                print(f"Executing: {build_command}")
+            build_result = subprocess.run(build_command, shell=True, env=container_build_env)
             # TODO: check result in build_result.returncode
             print(f"Result is: {build_result}")
         else:

app/build_npms.py

@@ -0,0 +1,92 @@
+# Copyright © 2022 Cerc
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+# Builds or pulls containers for the system components
+
+# env vars:
+# CERC_REPO_BASE_DIR defaults to ~/cerc
+
+import os
+from decouple import config
+import click
+import importlib
+from python_on_whales import docker
+from .util import include_exclude_check
+
+@click.command()
+@click.option('--include', help="only build these packages")
+@click.option('--exclude', help="don\'t build these packages")
+@click.pass_context
+def command(ctx, include, exclude):
+    '''build the set of npm packages required for a complete stack'''
+
+    quiet = ctx.obj.quiet
+    verbose = ctx.obj.verbose
+    dry_run = ctx.obj.dry_run
+    local_stack = ctx.obj.local_stack
+    debug = ctx.obj.debug
+
+    if local_stack:
+        dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
+        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
+    else:
+        dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
+
+    if not quiet:
+        print(f'Dev Root is: {dev_root_path}')
+
+    if not os.path.isdir(dev_root_path):
+        print('Dev root directory doesn\'t exist, creating')
+
+    # See: https://stackoverflow.com/a/20885799/1701505
+    from . import data
+    with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file:
+        packages = package_list_file.read().splitlines()
+
+    if verbose:
+        print(f'Packages: {packages}')
+
+    def build_package(package):
+        if not quiet:
+            print(f"Building npm package: {package}")
+        repo_dir = package
+        repo_full_path = os.path.join(dev_root_path, repo_dir)
+        # TODO: make the npm registry url configurable.
+        build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://gitea.local:3000/api/packages/cerc-io/npm/"]
+        if not dry_run:
+            if verbose:
+                print(f"Executing: {build_command}")
+            envs = {"CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"]} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
+            build_result = docker.run("cerc/builder-js",
+                                      remove=True,
+                                      interactive=True,
+                                      tty=True,
+                                      user=f"{os.getuid()}:{os.getgid()}",
+                                      envs=envs,
+                                      add_hosts=[("gitea.local", "host-gateway")],
+                                      volumes=[(repo_full_path, "/workspace")],
+                                      command=build_command
+                                      )
+            # TODO: check result in build_result.returncode
+            print(f"Result is: {build_result}")
+        else:
+            print("Skipped")
+
+    for package in packages:
+        if include_exclude_check(package, include, exclude):
+            build_package(package)
+        else:
+            if verbose:
+                print(f"Excluding: {package}")

app/data/compose/docker-compose-eth-probe.yml

@@ -0,0 +1,59 @@
+version: '3.2'
+
+services:
+  eth-probe-db:
+    restart: always
+    image: timescale/timescaledb:latest-pg14
+    environment:
+      POSTGRES_USER: "probe"
+      POSTGRES_DB: "probe"
+      POSTGRES_PASSWORD: "probe"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+    volumes:
+      - ../../eth-probe/db/schema.sql:/docker-entrypoint-initdb.d/init.sql
+    ports:
+      - 5432
+  eth-probe-mq:
+    restart: always
+    image: cerc/eth-probe:local
+    environment:
+      MODE: "mq"
+      PROBE_DEV: "false"
+      PGPORT: 5432
+      PGPASSWORD: "probe"
+      DB_USER: "probe"
+      PROBE_DB_NAME: "probe"
+      PROBE_DB_LOCATION: "eth-probe-db"
+      MQ_HOST: "0.0.0.0"
+      MQ_PORT: 33333
+      DEBUG: "vulcanize:*,cerc:*"
+    healthcheck:
+      test: [ "CMD", "nc", "-v", "localhost", "33333" ]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+    depends_on:
+      eth-probe-db:
+        condition: service_healthy
+  eth-probe-probe:
+    restart: always
+    image: cerc/eth-probe:local
+    environment:
+      MODE: "probe"
+      PROBE_DEV: "false"
+      MQ_HOST: "eth-probe-mq"
+      MQ_PORT: 33333
+      PROBE_ID: 0
+      GETH_HOST: "fixturenet-eth-geth-1"
+      GETH_MIN_BLOCK: 5
+      GETHJSON_URL: "http://fixturenet-eth-geth-1:9898/geth.json"
+      DEBUG: "vulcanize:*,cerc:*,-vulcanize:sniffer:dpt:error"
+    depends_on:
+      eth-probe-mq:
+        condition: service_healthy

app/data/compose/docker-compose-fixturenet-eth.yml

@@ -8,6 +8,9 @@ services:
     environment:
       RUN_BOOTNODE: "true"
     image: cerc/fixturenet-eth-geth:local
+    ports:
+      - "9898"
+      - "30303"
 
   fixturenet-eth-geth-1:
     hostname: fixturenet-eth-geth-1
@@ -28,7 +31,7 @@ services:
   fixturenet-eth-geth-2:
     hostname: fixturenet-eth-geth-2
     environment:
-      CERC_RUN_STATEDIFF: "true"
+      CERC_RUN_STATEDIFF: "detect"
       CERC_STATEDIFF_DB_NODE_ID: 2
     healthcheck:
       test: ["CMD", "nc", "-v", "localhost", "8545"]
@@ -40,7 +43,6 @@ services:
       - ../config/fixturenet-eth/fixturenet-eth.env
     image: cerc/fixturenet-eth-geth:local
     depends_on:
-      - migrations
       - fixturenet-eth-bootnode-geth
 
   fixturenet-eth-bootnode-lighthouse:

app/data/compose/docker-compose-ipld-eth-server.yml

@@ -30,4 +30,9 @@ services:
     ports:
       - "127.0.0.1:8081:8081"
       - "127.0.0.1:8082:8082"
-
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "8081"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s

app/data/compose/docker-compose-keycloak.yml

@@ -0,0 +1,37 @@
+version: '3.8'
+
+services:
+  keycloak-db:
+    image: postgres:14-alpine
+    env_file:
+      - ../config/keycloak/keycloak.env
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
+    ports:
+      - 5432
+  keycloak:
+    image: cerc/keycloak:local
+    env_file:
+      - ../config/keycloak/keycloak.env
+    environment:
+      JAVA_OPTS_APPEND: "-Dkeycloak.migration.action=import -Dkeycloak.migration.provider=dir -Dkeycloak.migration.dir=/import -Dkeycloak.migration.strategy=IGNORE_EXISTING"
+    volumes:
+      - ../config/keycloak/import:/import
+    ports:
+      - 8080
+    command: ["start"]
+    depends_on:
+      keycloak-db:
+        condition: service_healthy
+  keycloak-nginx:
+    image: nginx:1.23-alpine
+    volumes:
+      - ../config/keycloak/nginx:/etc/nginx/conf.d
+    ports:
+      - 80
+    depends_on:
+      - keycloak

app/data/compose/docker-compose-tx-spammer.yml

@@ -0,0 +1,18 @@
+version: '3.2'
+
+services:
+  tx-spammer:
+    restart: always
+    image: cerc/tx-spammer:local
+    env_file:
+      - ../config/tx-spammer/tx-spammer.env
+    environment:
+      ACCOUNTS_CSV_URL: http://fixturenet-eth-bootnode-geth:9898/accounts.csv
+      ETH_HTTP_PATH: http://fixturenet-eth-geth-1:8545
+      LOG_LEVEL: debug
+      SPAMMER_COMMAND: autoSend
+    depends_on:
+      fixturenet-eth-bootnode-geth:
+        condition: service_started
+      fixturenet-eth-geth-1:
+        condition: service_healthy

app/data/compose/docker-compose-watcher-erc20.yml

@@ -0,0 +1,49 @@
+version: '3.2'
+
+services:
+
+  erc20-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=erc20-watcher,erc20-watcher-job-queue
+      - POSTGRES_EXTENSION=erc20-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - erc20_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "0.0.0.0:15433:5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  erc20-watcher:
+    restart: unless-stopped
+    depends_on:
+      ipld-eth-server:
+        condition: service_healthy
+      erc20-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-erc20:local
+    environment:
+      - ETH_RPC_URL=http://go-ethereum:8545
+    command: ["sh", "-c", "yarn server"]
+    volumes:
+      - ../config/watcher-erc20/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml
+    ports:
+      - "0.0.0.0:3002:3001"
+      - "0.0.0.0:9002:9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3002"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+
+volumes:
+  erc20_watcher_db_data:

app/data/compose/docker-compose-watcher-erc721.yml

@@ -0,0 +1,49 @@
+version: '3.2'
+
+services:
+
+  erc721-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=erc721-watcher,erc721-watcher-job-queue
+      - POSTGRES_EXTENSION=erc721-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - erc721_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "0.0.0.0:15434:5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  erc721-watcher:
+    restart: unless-stopped
+    depends_on:
+      ipld-eth-server:
+        condition: service_healthy
+      erc721-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-erc721:local
+    environment:
+      - ETH_RPC_URL=http://go-ethereum:8545
+    command: ["sh", "-c", "yarn server"]
+    volumes:
+      - ../config/watcher-erc721/erc721-watcher.toml:/app/packages/erc721-watcher/environments/local.toml
+    ports:
+      - "0.0.0.0:3009:3009"
+      - "0.0.0.0:9003:9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3009"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+
+volumes:
+  erc721_watcher_db_data:

app/data/compose/docker-compose-watcher-mobymask.yml

@@ -4,7 +4,7 @@ version: '3.2'
 
 services:
 
-  watcher-db:
+  mobymask-watcher-db:
     restart: unless-stopped
     image: postgres:14-alpine
     environment:
@@ -14,7 +14,7 @@ services:
       - POSTGRES_PASSWORD=password
     volumes:
       - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
-      - watcher_db_data:/var/lib/postgresql/data
+      - mobymask_watcher_db_data:/var/lib/postgresql/data
     ports:
       - "0.0.0.0:15432:5432"
     healthcheck:
@@ -27,12 +27,12 @@ services:
   mobymask-watcher-server:
     restart: unless-stopped
     depends_on:
-      watcher-db:
+      mobymask-watcher-db:
         condition: service_healthy
     image: cerc/watcher-mobymask:local
     command: ["sh", "-c", "yarn server"]
     volumes:
-      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/watcher-ts/packages/mobymask-watcher/environments/local.toml
+      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml
     ports:
       - "0.0.0.0:3001:3001"
       - "0.0.0.0:9001:9001"
@@ -50,18 +50,16 @@ services:
     depends_on:
       mobymask-watcher-server:
         condition: service_healthy
-      watcher-db:
+      mobymask-watcher-db:
         condition: service_healthy
     image: cerc/watcher-mobymask:local
     command: ["sh", "-c", "yarn job-runner"]
     volumes:
-      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/watcher-ts/packages/mobymask-watcher/environments/local.toml
+      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml
     ports:
       - "0.0.0.0:9000:9000"
     extra_hosts:
       - "ipld-eth-server:host-gateway"
 
 volumes:
-  indexer_db_data:
-  watcher_db_data:
-  
+  mobymask_watcher_db_data:

app/data/compose/docker-compose-watcher-uniswap-v3.yml

@@ -0,0 +1,170 @@
+version: '3.2'
+
+services:
+
+  uniswap-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=erc20-watcher,uni-watcher,uni-info-watcher,erc20-watcher-job-queue,uni-watcher-job-queue,uni-info-watcher-job-queue
+      - POSTGRES_EXTENSION=erc20-watcher-job-queue:pgcrypto,uni-watcher-job-queue:pgcrypto,uni-info-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "work_mem=2GB"]
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - ../config/postgresql/create-pg-stat-statements.sql:/docker-entrypoint-initdb.d/create-pg-stat-statements.sql
+      - uniswap_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "0.0.0.0:15435:5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+    shm_size: '8GB'
+
+  erc20-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    working_dir: /app/packages/erc20-watcher
+    environment:
+      - DEBUG=vulcanize:*
+    command: ["node", "--enable-source-maps", "dist/server.js"]
+    volumes:
+      - ../config/watcher-uniswap-v3/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml
+    ports:
+      - "0.0.0.0:3005:3001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3001"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uni-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    working_dir: /app/packages/uni-watcher
+    environment:
+      - DEBUG=vulcanize:*
+    command: ["sh", "-c", "./watch-contract.sh && node --enable-source-maps dist/job-runner.js"]
+    volumes:
+      - ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml
+      - ../config/watcher-uniswap-v3/watch-contract.sh:/app/packages/uni-watcher/watch-contract.sh
+    ports:
+      - "0.0.0.0:9004:9000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uni-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+      uni-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    environment:
+      - UNISWAP_START_BLOCK=12369621
+      - DEBUG=vulcanize:*
+    working_dir: /app/packages/uni-watcher
+    command: ["./run.sh"]
+    volumes:
+      - ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml
+      - ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-watcher/run.sh
+    ports:
+      - "0.0.0.0:3003:3003"
+      - "0.0.0.0:9005:9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3003"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uni-info-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+      erc20-watcher-server:
+        condition: service_healthy
+      uni-watcher-server:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    working_dir: /app/packages/uni-info-watcher
+    environment:
+      - DEBUG=vulcanize:*
+    command: ["node", "--enable-source-maps", "dist/job-runner.js"]
+    volumes:
+      - ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml
+    ports:
+      - "0.0.0.0:9006:9002"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9002"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uni-info-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+      erc20-watcher-server:
+        condition: service_healthy
+      uni-watcher-server:
+        condition: service_healthy
+      uni-info-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    environment:
+      - UNISWAP_START_BLOCK=12369621
+    working_dir: /app/packages/uni-info-watcher
+    command: ["./run.sh"]
+    volumes:
+      - ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml
+      - ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-info-watcher/run.sh
+    ports:
+      - "0.0.0.0:3004:3004"
+      - "0.0.0.0:9007:9003"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3004"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uniswap-v3-info:
+    depends_on:
+      uni-info-watcher-server:
+        condition: service_healthy
+    image: cerc/uniswap-v3-info:local
+    ports:
+      - "0.0.0.0:3006:3000"
+
+volumes:
+  uniswap_watcher_db_data:

app/data/config/fixturenet-eth/fixturenet-eth.env

@@ -2,6 +2,7 @@
 ACCOUNT_PASSWORD=secret1212
 
 # ENODE of the geth bootnode.
+BOOTNODE_KEY="b0ac22adcad37213c7c565810a50f1772291e7b0ce53fb73e7ec2a3c75bc13b5"
 ENODE="enode://af22c29c316ad069cf48a09a4ad5cf04a251b411e45098888d114c6dd7f489a13786620d5953738762afa13711d4ffb3b19aa5de772d8af72f851f7e9c5b164a@fixturenet-eth-bootnode-geth:30303"
 
 # JWT shared by geth and lighthouse for authentication.
@@ -16,3 +17,4 @@ CERC_STATEDIFF_DB_PORT=5432
 CERC_STATEDIFF_DB_NAME="cerc_testing"
 CERC_STATEDIFF_DB_USER="vdbm"
 CERC_STATEDIFF_DB_PASSWORD="password"
+CERC_STATEDIFF_DB_GOOSE_MIN_VER=23

app/data/config/keycloak/keycloak.env

@@ -0,0 +1,17 @@
+POSTGRES_DB=keycloak
+POSTGRES_USER=keycloak
+POSTGRES_PASSWORD=keycloak
+KC_DB=postgres
+KC_DB_URL_HOST=keycloak-db
+KC_DB_URL_DATABASE=${POSTGRES_DB}
+KC_DB_USERNAME=${POSTGRES_USER}
+KC_DB_PASSWORD=${POSTGRES_PASSWORD}
+KC_DB_SCHEMA=public
+KC_HOSTNAME=localhost
+KC_HTTP_ENABLED="true"
+KC_HTTP_RELATIVE_PATH="/auth"
+KC_HOSTNAME_STRICT_HTTPS="false"
+KEYCLOAK_ADMIN=admin
+KEYCLOAK_ADMIN_PASSWORD=admin
+X_API_CHECK_REALM=cerc
+X_API_CHECK_CLIENT_ID="%api_key%"

app/data/config/keycloak/nginx/keycloak_proxy.conf

@@ -0,0 +1,69 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  localhost;
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+
+### geth
+    location ~ ^/eth/?([^/]*)$ {
+        set $apiKey $1;
+        if ($apiKey = '') {
+          set $apiKey $http_X_API_KEY;
+        }
+        auth_request /auth;
+        proxy_buffering off;
+        rewrite /.*$ / break;
+        proxy_pass  http://fixturenet-eth-geth-1:8545;
+    }
+
+### ipld-eth-server
+#    location ~ ^/ipld/eth/([^/]*)$ {
+#        set $apiKey $1;
+#        if ($apiKey = '') {
+#          set $apiKey $http_X_API_KEY;
+#        }
+#        auth_request /auth;
+#        proxy_buffering off;
+#        rewrite /.*$ / break;
+#        proxy_pass  http://ipld-eth-server:8081;
+#    }
+#
+#    location ~ ^/ipld/gql/([^/]*)$ {
+#        set $apiKey $1;
+#        if ($apiKey = '') {
+#          set $apiKey $http_X_API_KEY;
+#        }
+#        auth_request /auth;
+#        proxy_buffering off;
+#        rewrite /.*$ / break;
+#        proxy_pass  http://ipld-eth-server:8082;
+#    }
+#
+### lighthouse
+#    location /beacon/ {
+#        set $apiKey $http_X_API_KEY;
+#        auth_request /auth;
+#        proxy_buffering off;
+#        proxy_pass  http://fixturenet-eth-lighthouse-1:8001/;
+#    }
+
+    location = /auth {
+        internal;
+        proxy_buffering off;
+        resolver 127.0.0.11 ipv6=off;
+        proxy_pass http://keycloak:8080/auth/realms/cerc/check?apiKey=$apiKey;
+        proxy_pass_request_body off;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Original-Remote-Addr $remote_addr;
+        proxy_set_header X-Original-Host $host;
+    }
+
+#    location = /basic_status {
+#        stub_status;
+#    }
+}

app/data/config/postgresql/create-pg-stat-statements.sql

@@ -0,0 +1 @@
+CREATE EXTENSION pg_stat_statements;

app/data/config/tx-spammer/tx-spammer.env

@@ -0,0 +1,2 @@
+ETH_CALL_FREQ=1000
+ETH_SEND_FREQ=1000

app/data/config/watcher-erc20/erc20-watcher.toml

@@ -0,0 +1,41 @@
+[server]
+  host = "0.0.0.0"
+  port = 3001
+  mode = "storage"
+  kind = "lazy"
+
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "erc20-watcher-db"
+  port = 5432
+  database = "erc20-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+  maxQueryExecutionTime = 100
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server:8082/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server:8081"
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@erc20-watcher-db:5432/erc20-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  blockDelayInMilliSecs = 2000
+  prefetchBlocksInMem = true
+  prefetchBlockCount = 10

app/data/config/watcher-erc721/erc721-watcher.toml

@@ -0,0 +1,56 @@
+[server]
+  host = "0.0.0.0"
+  port = 3009
+  kind = "lazy"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  enableState = true
+
+  # Boolean to filter logs by contract.
+  filterLogs = false
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "erc721-watcher-db"
+  port = 5432
+  database = "erc721-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+  maxQueryExecutionTime = 100
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server:8082/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server:8081"
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@erc721-watcher-db:5432/erc721-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  blockDelayInMilliSecs = 2000
+  prefetchBlocksInMem = true
+  prefetchBlockCount = 10

app/data/config/watcher-mobymask/mobymask-watcher.toml

@@ -27,7 +27,7 @@
 
 [database]
   type = "postgres"
-  host = "watcher-db"
+  host = "mobymask-watcher-db"
   port = 5432
   database = "mobymask-watcher"
   username = "vdbm"
@@ -47,7 +47,7 @@
     deleteOnStart = false
 
 [jobQueue]
-  dbConnectionString = "postgres://vdbm:password@watcher-db/mobymask-watcher-job-queue"
+  dbConnectionString = "postgres://vdbm:password@mobymask-watcher-db/mobymask-watcher-job-queue"
   maxCompletionLagInSecs = 300
   jobDelayInMilliSecs = 100
   eventsInBatch = 50

app/data/config/watcher-uniswap-v3/erc20-watcher.toml

@@ -0,0 +1,39 @@
+[server]
+  host = "0.0.0.0"
+  port = 3001
+  mode = "eth_call"
+  kind = "lazy"
+
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+    [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "uniswap-watcher-db"
+  port = 5432
+  database = "erc20-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+  maxQueryExecutionTime = 100
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/erc20-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  blockDelayInMilliSecs = 2000

app/data/config/watcher-uniswap-v3/run.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+set -u
+
+echo "Initializing watcher..."
+yarn fill --start-block $UNISWAP_START_BLOCK --end-block $((UNISWAP_START_BLOCK + 1))
+
+echo "Running active server"
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

app/data/config/watcher-uniswap-v3/uni-info-watcher.toml

@@ -0,0 +1,90 @@
+[server]
+  host = "0.0.0.0"
+  port = 3004
+  mode = "prod"
+  kind = "active"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 50000
+
+  # Enable state creation
+  enableState = false
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Boolean to skip updating entity fields required in state creation and not required in the frontend.
+  skipStateFieldsUpdate = false
+
+  # Boolean to load GQL query nested entity relations sequentially.
+  loadRelationsSequential = false
+
+  # Max GQL API requests to process simultaneously (defaults to 1).
+  maxSimultaneousRequests = 1
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9002
+  [metrics.gql]
+    port = 9003
+
+[database]
+  type = "postgres"
+  host = "uniswap-watcher-db"
+  port = 5432
+  database = "uni-info-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+  maxQueryExecutionTime = 100
+
+  [database.extra]
+    # maximum number of clients the pool should contain
+    max = 20
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+  [upstream.uniWatcher]
+    gqlEndpoint = "http://uni-watcher-server:3003/graphql"
+    gqlSubscriptionEndpoint = "ws://uni-watcher-server:3003/graphql"
+
+  [upstream.tokenWatcher]
+    gqlEndpoint = "http://erc20-watcher-server:3001/graphql"
+    gqlSubscriptionEndpoint = "ws://erc20-watcher-server:3001/graphql"
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-info-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 1000
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  blockDelayInMilliSecs = 2000
+  prefetchBlocksInMem = true
+  prefetchBlockCount = 10

app/data/config/watcher-uniswap-v3/uni-watcher.toml

@@ -0,0 +1,41 @@
+[server]
+  host = "0.0.0.0"
+  port = 3003
+  kind = "active"
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "uniswap-watcher-db"
+  port = 5432
+  database = "uni-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+  maxQueryExecutionTime = 100
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 0
+  eventsInBatch = 50
+  lazyUpdateBlockProgress = true
+  blockDelayInMilliSecs = 2000
+  prefetchBlocksInMem = true
+  prefetchBlockCount = 10

app/data/config/watcher-uniswap-v3/watch-contract.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+set -u
+
+echo "Watching factory contract 0x1F98431c8aD98523631AE4a59f267346ea31F984"
+yarn watch:contract --address 0x1F98431c8aD98523631AE4a59f267346ea31F984 --kind factory --startingBlock 12369621 --checkpoint
+
+echo "Watching nfpm contract 0xC36442b4a4522E871399CD717aBDD847Ab11FE88"
+yarn watch:contract --address 0xC36442b4a4522E871399CD717aBDD847Ab11FE88 --kind nfpm --startingBlock 12369651 --checkpoint

app/data/container-build/cerc-builder-js/Dockerfile

@@ -0,0 +1,48 @@
+# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT=16-bullseye
+FROM node:${VARIANT}
+
+ARG USERNAME=node
+ARG NPM_GLOBAL=/usr/local/share/npm-global
+
+# Add NPM global to PATH.
+ENV PATH=${NPM_GLOBAL}/bin:${PATH}
+
+RUN \
+    # Configure global npm install location, use group to adapt to UID/GID changes
+    if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \
+    && usermod -a -G npm ${USERNAME} \
+    && umask 0002 \
+    && mkdir -p ${NPM_GLOBAL} \
+    && touch /usr/local/etc/npmrc \
+    && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \
+    && chmod g+s ${NPM_GLOBAL} \
+    && npm config -g set prefix ${NPM_GLOBAL} \
+    && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
+    # Install eslint
+    && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+    && npm cache clean --force > /dev/null 2>&1
+
+# [Optional] Uncomment this section to install additional OS packages.
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends jq
+
+# [Optional] Uncomment if you want to install an additional version of node using nvm
+# ARG EXTRA_NODE_VERSION=10
+# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
+
+# [Optional] Uncomment if you want to install more global node modules
+# RUN su node -c "npm install -g <your-package-list-here>"
+
+RUN mkdir /scripts
+COPY build-npm-package.sh /scripts
+COPY yarn-local-registry-fixup.sh /scripts
+COPY build-npm-package-local-dependencies.sh /scripts
+ENV PATH="${PATH}:/scripts"
+
+COPY entrypoint.sh .
+ENTRYPOINT ["./entrypoint.sh"]
+# Placeholder CMD : generally this will be overridden at run time like :
+# docker run -it -v /home/builder/cerc/laconic-sdk:/workspace cerc/builder-js sh -c 'cd /workspace && yarn && yarn build'
+CMD node --version

app/data/container-build/cerc-builder-js/README.md

@@ -0,0 +1,17 @@
+## JS/TS Package Builder
+
+This container is designed to be used as a simple "build runner" environment for building and publishing JS/TS projects
+using `yarn`.
+
+### Running a build
+
+As a temporary measure while the necessary functionality is being added to Stack Orchestrator,
+it is possible to build packages manually by invoking `docker run` , for example as follows:
+
+
+```
+$ docker run --rm -it --add-host gitea.local:host-gateway \
+  -v ${HOME}/cerc/laconic-registry-cli:/workspace cerc/builder-js  \
+  sh -c 'cd /workspace && CERC_NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \
+  build-npm-package-local-dependencies.sh http://gitea.local:3000/api/packages/cerc-io/npm/ 0.1.8'
+```

app/data/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Usage: build-npm-package-local-dependencies.sh <registry-url> <publish-with-this-version>
+# Runs build-npm-package.sh after first fixing up yarn.lock to use a local
+# npm registry for all packages in a spcific scope (currently @cerc-io)
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+if ! [[ $# -eq 1 || $# -eq 2 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+if [[ -z "${CERC_NPM_AUTH_TOKEN}" ]]; then
+    echo "CERC_NPM_AUTH_TOKEN is not set" >&2
+    exit 1
+fi
+local_npm_registry_url=$1
+package_publish_version=$2
+# TODO: make this a paramater and allow a list of scopes
+npm_scope_for_local="@cerc-io"
+# We need to configure the local registry 
+npm config set ${npm_scope_for_local}:registry ${local_npm_registry_url}
+npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN}
+# Find the set of dependencies from the specified scope
+mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | with_entries(if (.key|test("^'${npm_scope_for_local}'/.*$")) then ( {key: .key, value: .value } ) else empty end ) | keys[]')
+echo "Fixing up dependencies"
+for package in "${dependencies_from_scope[@]}"
+do
+    echo "Fixing up package ${package}"
+    yarn-local-registry-fixup.sh $package ${local_npm_registry_url}
+done
+echo "Running build"
+build-npm-package.sh ${local_npm_registry_url} ${package_publish_version}

app/data/container-build/cerc-builder-js/build-npm-package.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Usage: build-npm-package.sh <registry-url> <publish-with-this-version>
+# Note: supply the registry auth token in CERC_NPM_AUTH_TOKEN
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+if ! [[ $# -eq 1 || $# -eq 2 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+if [[ -z "${CERC_NPM_AUTH_TOKEN}" ]]; then
+    echo "CERC_NPM_AUTH_TOKEN is not set" >&2
+    exit 1
+fi
+if [[ $# -eq 2 ]]; then
+    package_publish_version=$2
+else
+    package_publish_version=$( cat package.json | jq -r .version )
+fi
+# Get the name of this package from package.json since we weren't passed that
+package_name=$( cat package.json | jq -r .name )
+local_npm_registry_url=$1
+npm config set @lirewine:registry ${local_npm_registry_url}
+npm config set @cerc-io:registry ${local_npm_registry_url}
+npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN}
+# First check if the version of this package we're trying to build already exists in the registry
+package_exists=$( yarn info --json ${package_name}@${package_publish_version} | jq -r .data.dist.tarball )
+if [[ ! -z "$package_exists" && "$package_exists" != "null" ]]; then
+    echo "${package_publish_version} of ${package_name} already exists in the registry, skipping build"
+    exit 0
+fi
+echo "Build and publish ${package_name} version ${package_publish_version}"
+yarn install
+yarn build
+yarn publish --non-interactive  --new-version ${package_publish_version} --no-git-tag-version

app/data/container-build/cerc-builder-js/entrypoint.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec "$@"

app/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Usage: yarn-local-registry-fixup.sh <package-to-fix> <registry-url>
+# Assumes package.json and yarn.lock are in the cwd
+# The purpose of this script is to take a project cloned from git
+# and "fixup" its yarn.lock file such that specified dependency
+# will be fetched from a registry other than the one used when
+# yarn.lock was generated. It updates all checksums using data
+# from the "new" registry (because due to embedded timestamps etc
+# the same source code re-built later will not have the same checksum).
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+if [[ $# -ne 2 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+target_package=$1
+local_npm_registry_url=$2
+# TODO: use jq rather than sed here:
+versioned_target_package=$(grep ${target_package} package.json | sed -e 's#[[:space:]]\{1,\}\"\('${target_package}'\)\":[[:space:]]\{1,\}\"\(.*\)\",#\1@\2#' )
+# Use yarn info to get URL checksums etc from the new registry
+yarn_info_output=$(yarn info --json $versioned_target_package 2>/dev/null)
+# Code below parses out the values we need
+package_tarball=$(echo $yarn_info_output | jq -r .data.dist.tarball)
+# When running inside a container, the registry can return a URL with the wrong host name due to proxying
+# so we need to check if that has happened and fix the URL if so.
+if ! [[ "${package_tarball}" =~ ^${local_npm_registry_url}.* ]]; then
+    # HACK: I've hard-wired the host names below. Replace with proper implementation
+    # TODO: remove the hack when proven no longer necessary
+    package_tarball=$( echo ${package_tarball} | sed -e 's/localhost/gitea.local/g' )
+fi
+package_integrity=$(echo $yarn_info_output | jq -r .data.dist.integrity)
+package_shasum=$(echo $yarn_info_output | jq -r .data.dist.shasum)
+package_resolved=${package_tarball}#${package_shasum}
+# Some strings need to be escaped so they work when passed to sed later
+escaped_package_resolved=$(printf '%s\n' "$package_resolved" | sed -e 's/[\/&]/\\&/g')
+escaped_target_package=$(printf '%s\n' "$target_package" | sed -e 's/[\/&]/\\&/g')
+if [ -n "$CERC_SCRIPT_VERBOSE" ]; then
+    echo "Tarball: ${package_tarball}"
+    echo "Integrity: ${package_integrity}"
+    echo "Shasum: ${package_shasum}"
+    echo "Resolved: ${package_resolved}"
+fi
+# Use magic sed regex to replace the values in yarn.lock
+# Note: yarn.lock is not json so we can not use jq for this
+sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}resolved \).*$/\1'\"${escaped_package_resolved}\"'/' yarn.lock
+sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}integrity \).*$/\1'${package_integrity}'/' yarn.lock

app/data/container-build/cerc-eth-probe/build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# Build cerc/eth-probe
+docker build -t cerc/eth-probe:local ${CERC_REPO_BASE_DIR}/eth-probe

app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile

@@ -1,7 +1,7 @@
 FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
 
 FROM cerc/go-ethereum:local
-RUN apk add --no-cache python3 python3-dev py3-pip curl wget jq build-base gettext libintl openssl bash
+RUN apk add --no-cache python3 python3-dev py3-pip curl wget jq build-base gettext libintl openssl bash bind-tools postgresql-client
 
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/eth2-testnet-genesis
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/eth2-val-tools

app/data/container-build/cerc-fixturenet-eth-geth/genesis/el/build_el.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -e
+
+# See: https://github.com/skylenet/ethereum-genesis-generator/blob/master/entrypoint.sh
+
+rm -rf ../build/el
+mkdir -p ../build/el
+
+tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
+envsubst < el-config.yaml > $tmp_dir/genesis-config.yaml
+
+ttd=`cat $tmp_dir/genesis-config.yaml | grep terminal_total_difficulty | awk '{ print $2 }'`
+homestead_block=`cat $tmp_dir/genesis-config.yaml | grep homestead_block | awk '{ print $2 }'`
+eip150_block=`cat $tmp_dir/genesis-config.yaml | grep eip150_block | awk '{ print $2 }'`
+eip155_block=`cat $tmp_dir/genesis-config.yaml | grep eip155_block | awk '{ print $2 }'`
+eip158_block=`cat $tmp_dir/genesis-config.yaml | grep eip158_block | awk '{ print $2 }'`
+byzantium_block=`cat $tmp_dir/genesis-config.yaml | grep byzantium_block | awk '{ print $2 }'`
+constantinople_block=`cat $tmp_dir/genesis-config.yaml | grep constantinople_block | awk '{ print $2 }'`
+petersburg_block=`cat $tmp_dir/genesis-config.yaml | grep petersburg_block | awk '{ print $2 }'`
+istanbul_block=`cat $tmp_dir/genesis-config.yaml | grep istanbul_block | awk '{ print $2 }'`
+berlin_block=`cat $tmp_dir/genesis-config.yaml | grep berlin_block | awk '{ print $2 }'`
+london_block=`cat $tmp_dir/genesis-config.yaml | grep london_block | awk '{ print $2 }'`
+merge_fork_block=`cat $tmp_dir/genesis-config.yaml | grep merge_fork_block | awk '{ print $2 }'`
+
+python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \
+  jq ".config.terminalTotalDifficulty=$ttd" | \
+  jq ".config.homesteadBlock=$homestead_block" | \
+  jq ".config.eip150Block=$eip150_block" | \
+  jq ".config.eip155Block=$eip155_block" | \
+  jq ".config.eip158Block=$eip158_block" | \
+  jq ".config.byzantiumBlock=$byzantium_block" | \
+  jq ".config.constantinopleBlock=$constantinople_block" | \
+  jq ".config.petersburgBlock=$petersburg_block" | \
+  jq ".config.istanbulBlock=$istanbul_block" | \
+  jq ".config.berlinBlock=$berlin_block" | \
+  jq ".config.londonBlock=$london_block" | \
+  jq ".config.mergeForkBlock=$merge_fork_block" > ../build/el/geth.json
+python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv

app/data/container-build/cerc-fixturenet-eth-geth/genesis/el/el-config.yaml

@@ -11,6 +11,17 @@ chain_id: 1212
 deposit_contract_address: "0x1212121212121212121212121212121212121212"
 genesis_timestamp: 0
 terminal_total_difficulty: 1000
+homestead_block: 1
+eip150_block: 1
+eip155_block: 1
+eip158_block: 1
+byzantium_block: 1
+constantinople_block: 1
+petersburg_block: 1
+istanbul_block: 1
+berlin_block: 1
+london_block: 1
+merge_fork_block: 1
 
 clique:
   enabled: false

app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -4,9 +4,14 @@ ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
 NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
 NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
 
+HOME_DIR=`pwd`
+cd /opt/testnet/build/el
+python3 -m http.server 9898 &
+cd $HOME_DIR
+
 if [ "true" == "$RUN_BOOTNODE" ]; then 
     geth \
-      --nodekeyhex="b0ac22adcad37213c7c565810a50f1772291e7b0ce53fb73e7ec2a3c75bc13b5" \
+      --nodekeyhex="${BOOTNODE_KEY}" \
       --nodiscover \
       --ipcdisable \
       --networkid=${NETWORK_ID} \
@@ -17,8 +22,33 @@ else
     
     echo -n "$JWT" > /opt/testnet/build/el/jwtsecret
 
+    if [ "$CERC_RUN_STATEDIFF" == "detect" ] && [ -n "$CERC_STATEDIFF_DB_HOST" ]; then
+      if [ -n "$(dig $CERC_STATEDIFF_DB_HOST +short)" ]; then
+        echo "Statediff DB at $CERC_STATEDIFF_DB_HOST"
+        CERC_RUN_STATEDIFF="true"
+      else
+        echo "No statediff DB available."
+        CERC_RUN_STATEDIFF="false"
+      fi
+    fi
+
     STATEDIFF_OPTS=""
     if [ "$CERC_RUN_STATEDIFF" == "true" ]; then
+      ready=0
+      while [ $ready -eq 0 ]; do
+        echo "Waiting for statediff DB..."
+        sleep 1
+        export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD"
+        result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \
+          -p "$CERC_STATEDIFF_DB_PORT" \
+          -U "$CERC_STATEDIFF_DB_USER" \
+          -d "$CERC_STATEDIFF_DB_NAME" \
+          -t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }')
+        if [ -n "$result" ] && [ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]; then
+          echo "DB ready..."
+          ready=1
+        fi
+      done
       STATEDIFF_OPTS="--statediff=true \
       --statediff.db.host=$CERC_STATEDIFF_DB_HOST \
       --statediff.db.name=$CERC_STATEDIFF_DB_NAME \
@@ -49,5 +79,6 @@ else
       --syncmode=full \
       --mine \
       --miner.threads=1 \
+      --verbosity=5 \
       --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS} 2>&1 | tee /var/log/geth.log
 fi

app/data/container-build/cerc-go-ethereum-foundry/genesis-automine.json

@@ -10,7 +10,7 @@
     "petersburgBlock": 0,
     "istanbulBlock": 0,
     "clique": {
-      "period": 2,
+      "period": 5,
       "epoch": 3000
     }
   },

app/data/container-build/cerc-keycloak/Dockerfile

@@ -0,0 +1,4 @@
+FROM quay.io/keycloak/keycloak:20.0
+WORKDIR /opt/keycloak/providers
+RUN curl -L https://github.com/aerogear/keycloak-metrics-spi/releases/download/2.5.3/keycloak-metrics-spi-2.5.3.jar --output keycloak-metrics-spi.jar
+RUN curl -L https://github.com/cerc-io/keycloak-api-key-demo/releases/download/v0.1/api-key-module-0.1.jar --output api-key-module.jar

app/data/container-build/cerc-keycloak/build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+#
+#Build cerc/keycloack
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/keycloak:local ${SCRIPT_DIR}

app/data/container-build/cerc-laconic-registry-cli/Dockerfile

@@ -0,0 +1,57 @@
+# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT=16-bullseye
+FROM node:${VARIANT}
+
+ARG USERNAME=node
+ARG NPM_GLOBAL=/usr/local/share/npm-global
+
+# This container pulls npm packages from a local registry configured via these env vars
+ARG CERC_NPM_URL
+ARG CERC_NPM_AUTH_TOKEN
+
+# Add NPM global to PATH.
+ENV PATH=${NPM_GLOBAL}/bin:${PATH}
+
+RUN \
+    # Configure global npm install location, use group to adapt to UID/GID changes
+    if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \
+    && usermod -a -G npm ${USERNAME} \
+    && umask 0002 \
+    && mkdir -p ${NPM_GLOBAL} \
+    && touch /usr/local/etc/npmrc \
+    && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \
+    && chmod g+s ${NPM_GLOBAL} \
+    && npm config -g set prefix ${NPM_GLOBAL} \
+    && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
+    # Install eslint
+    && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+    && npm cache clean --force > /dev/null 2>&1
+
+# [Optional] Uncomment this section to install additional OS packages.
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends jq
+
+# [Optional] Uncomment if you want to install an additional version of node using nvm
+# ARG EXTRA_NODE_VERSION=10
+# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
+
+# [Optional] Uncomment if you want to install more global node modules
+# RUN su node -c "npm install -g <your-package-list-here>"
+
+# Configure the local npm registry
+RUN npm config set @lirewine:registry ${CERC_NPM_URL} \
+    && npm config set @cerc-io:registry ${CERC_NPM_URL} \
+    && npm config set @lirewine:registry ${CERC_NPM_URL} \
+    && npm config set -- ${CERC_NPM_URL}:_authToken ${CERC_NPM_AUTH_TOKEN}
+
+# TODO: the image at this point could be made a base image for several different CLI images
+# that install different Node-based CLI commands
+
+# DEBUG, remove
+RUN yarn info @cerc-io/laconic-registry-cli
+
+# Globally install the cli package
+RUN yarn global add @cerc-io/laconic-registry-cli
+
+ENTRYPOINT ["laconic"]

app/data/container-build/cerc-laconic-registry-cli/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/laconic-registry-cli
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/laconic-registry-cli:local -f ${SCRIPT_DIR}/Dockerfile \
+  --add-host gitea.local:host-gateway \
+  --build-arg CERC_NPM_AUTH_TOKEN --build-arg CERC_NPM_URL ${SCRIPT_DIR}