Add call to cleanup function (#486)

* Force a recent version of pydantic

* Really pin the version

* Really pin the version

app/data/compose/docker-compose-fixturenet-plugeth.yml

@@ -9,7 +9,6 @@ services:
     image: cerc/fixturenet-plugeth-plugeth:local
     volumes:
       - fixturenet_plugeth_bootnode_geth_data:/root/ethdata
-      - ../config/fixturenet-plugeth/plugins:/root/ethdata/plugins
     ports:
       - "9898"
       - "30303"
@@ -66,7 +65,7 @@ services:
     hostname: fixturenet-eth-bootnode-lighthouse
     environment:
       RUN_BOOTNODE: "true"
-    image: cerc/fixturenet-plugeth-lighthouse:local
+    image: cerc/fixturenet-eth-lighthouse:local
 
   fixturenet-eth-lighthouse-1:
     restart: always
@@ -83,7 +82,7 @@ services:
       NODE_NUMBER: "1"
       ETH1_ENDPOINT: "http://fixturenet-eth-geth-1:8545"
       EXECUTION_ENDPOINT: "http://fixturenet-eth-geth-1:8551"
-    image: cerc/fixturenet-plugeth-lighthouse:local
+    image: cerc/fixturenet-eth-lighthouse:local
     volumes:
       - fixturenet_plugeth_lighthouse_1_data:/opt/testnet/build/cl
     depends_on:
@@ -110,7 +109,7 @@ services:
       ETH1_ENDPOINT: "http://fixturenet-eth-geth-2:8545"
       EXECUTION_ENDPOINT: "http://fixturenet-eth-geth-2:8551"
       LIGHTHOUSE_GENESIS_STATE_URL: "http://fixturenet-eth-lighthouse-1:8001/eth/v2/debug/beacon/states/0"
-    image: cerc/fixturenet-plugeth-lighthouse:local
+    image: cerc/fixturenet-eth-lighthouse:local
     volumes:
       - fixturenet_plugeth_lighthouse_2_data:/opt/testnet/build/cl
     depends_on:

app/data/compose/docker-compose-mobymask-app.yml

@@ -14,12 +14,14 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_BUILD_DIR: "@cerc-io/mobymask-ui/build"
+      CERC_RELEASE: "v0.1.5"
+      CERC_USE_NPM: true
+      CERC_CONFIG_FILE: "src/config.json"
     working_dir: /scripts
     command: ["sh", "mobymask-app-start.sh"]
     volumes:
-      - ../config/network/wait-for-it.sh:/scripts/wait-for-it.sh
       - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
+      - ../config/watcher-mobymask-v2/mobymask-app-config.json:/app/src/mobymask-app-config.json
       - peers_ids:/peers
       - mobymask_deployment:/server
     ports:
@@ -46,12 +48,14 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_BUILD_DIR: "@cerc-io/mobymask-ui-lxdao/build"
+      CERC_RELEASE: "v0.1.5-lxdao-0.1.1"
+      CERC_USE_NPM: false
+      CERC_CONFIG_FILE: "src/utils/config.json"
     working_dir: /scripts
     command: ["sh", "mobymask-app-start.sh"]
     volumes:
-      - ../config/network/wait-for-it.sh:/scripts/wait-for-it.sh
       - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
+      - ../config/watcher-mobymask-v2/mobymask-app-config.json:/app/src/mobymask-app-config.json
       - peers_ids:/peers
       - mobymask_deployment:/server
     ports:

app/data/config/fixturenet-plugeth/plugins/README.md

@@ -1 +0,0 @@
-See: https://docs.plugeth.org/

app/data/config/mainnet-eth/scripts/run-geth.sh

@@ -23,6 +23,8 @@ cleanup() {
     echo "Done"
 }
 
+trap 'cleanup' SIGINT SIGTERM
+
 $START_CMD \
     --datadir="${CERC_ETH_DATADIR}" \
     --authrpc.addr="0.0.0.0" \

app/data/config/watcher-mobymask-v2/mobymask-app-config.json

@@ -0,0 +1,7 @@
+{
+  "name": "MobyMask",
+  "relayNodes": [],
+  "peer": {
+    "enableDebugInfo": true
+  }
+}

app/data/config/watcher-mobymask-v2/mobymask-app-start.sh

@@ -33,11 +33,23 @@ else
   echo "Taking deployed contract details from env"
 fi
 
-# Use yq to create config.yml with environment variables
-yq -n ".address = env(CERC_DEPLOYED_CONTRACT)" > /config/config.yml
-yq ".watcherUrl = env(CERC_APP_WATCHER_URL)" -i /config/config.yml
-yq ".chainId = env(CERC_CHAIN_ID)" -i /config/config.yml
-yq ".relayNodes = strenv(CERC_RELAY_NODES)" -i /config/config.yml
-yq ".denyMultiaddrs = strenv(CERC_DENY_MULTIADDRS)" -i /config/config.yml
+cd /app
+git checkout $CERC_RELEASE
 
-/scripts/start-serving-app.sh
+# Export config values in a json file
+jq --arg address "$CERC_DEPLOYED_CONTRACT" \
+  --argjson chainId "$CERC_CHAIN_ID" \
+  --argjson relayNodes "$CERC_RELAY_NODES" \
+  --argjson denyMultiaddrs "$CERC_DENY_MULTIADDRS" \
+  '.address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.denyMultiaddrs = $denyMultiaddrs' \
+  /app/src/mobymask-app-config.json > /app/${CERC_CONFIG_FILE}
+
+if [ "${CERC_USE_NPM}" = "true" ]; then
+  npm install
+  REACT_APP_WATCHER_URI="$CERC_APP_WATCHER_URL/graphql" npm run build
+else
+  yarn install
+  REACT_APP_WATCHER_URI="$CERC_APP_WATCHER_URL/graphql" yarn build
+fi
+
+http-server -p 80 /app/build

app/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -0,0 +1,27 @@
+FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
+
+FROM golang:1.20-alpine as builder
+
+RUN apk add --no-cache python3 py3-pip
+
+COPY genesis /opt/genesis
+
+# Install ethereum-genesis-generator tools
+COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
+COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
+COPY --from=ethgen /apps /apps
+RUN cd /apps/el-gen && pip3 install -r requirements.txt
+# web3==5.24.0 used by el-gen is broken on python 3.11
+RUN pip3 install --upgrade "web3==6.5.0"
+
+# Build genesis config
+RUN apk add --no-cache make bash envsubst jq
+RUN cd /opt/genesis && make genesis-el
+
+# Snag the genesis block info.
+RUN go install github.com/cerc-io/eth-dump-genblock@latest
+RUN eth-dump-genblock /opt/genesis/build/el/geth.json > /opt/genesis/build/el/genesis_block.json
+
+FROM alpine:latest
+
+COPY --from=builder /opt/genesis /opt/genesis

app/data/container-build/cerc-fixturenet-eth-genesis/build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Build cerc/fixturenet-eth-genesis
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/fixturenet-eth-genesis:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR

app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile

@@ -1,39 +1,21 @@
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
-
-# Using the same golang image as used to build geth: https://github.com/cerc-io/go-ethereum/blob/HEAD/Dockerfile
-FROM golang:1.18-alpine as delve
-RUN go install github.com/go-delve/delve/cmd/dlv@latest
-
+FROM cerc/fixturenet-eth-genesis:local as fnetgen
 FROM cerc/go-ethereum:local as geth
 
+# Using the same golang image as used to build geth: https://github.com/cerc-io/go-ethereum/blob/HEAD/Dockerfile
+FROM golang:1.20-alpine as delve
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
 FROM alpine:3.17
-RUN apk add --no-cache python3 python3-dev py3-pip curl wget jq build-base gettext libintl openssl bash bind-tools postgresql-client
 
-COPY --from=delve  /go/bin/dlv /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
-COPY --from=ethgen /apps /apps
+RUN apk add --no-cache bash wget python3 bind-tools postgresql-client
 
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
-
-COPY genesis /opt/testnet
 COPY run-el.sh /opt/testnet/run.sh
 
-RUN cd /opt/testnet && make genesis-el
+COPY --from=delve /go/bin/dlv /usr/local/bin/
+COPY --from=geth /usr/local/bin/geth /usr/local/bin/
+COPY --from=fnetgen /opt/genesis /opt/testnet
 
-COPY --from=geth  /usr/local/bin/geth /usr/local/bin/
-
-# Snag the genesis block info.
+# Initialize the geth db with our config
 RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
-RUN cp -rp ~/ethdata ~/tmpeth && \
-    geth --datadir ~/tmpeth init /opt/testnet/build/el/geth.json && \
-    geth --datadir ~/tmpeth --http & \
-    sleep 5 && \
-    curl -q --location 'localhost:8545' \
-        --header 'Content-Type: application/json' \
-        --data '{ "jsonrpc": "2.0", "id": 14, "method": "eth_getBlockByNumber", "params": ["0x0", false] }' \
-        -o /opt/testnet/build/el/genesis_block.json && \
-    killall -9 geth && \
-    rm -rf ~/tmpeth
 
 ENTRYPOINT ["/opt/testnet/run.sh"]

app/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile

@@ -1,6 +1,6 @@
 FROM cerc/lighthouse-cli:local AS lcli
 FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
-FROM cerc/fixturenet-eth-geth:local AS fnetgeth
+FROM cerc/fixturenet-eth-genesis:local AS fnetgen
 
 FROM cerc/lighthouse:local
 
@@ -19,8 +19,8 @@ COPY --from=lcli /usr/local/bin/lcli /usr/local/bin/lcli
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/eth2-testnet-genesis
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/eth2-val-tools
 COPY --from=ethgen /apps /apps
-COPY --from=fnetgeth /opt/testnet/el /opt/testnet/el
-COPY --from=fnetgeth /opt/testnet/build/el /opt/testnet/build/el
+COPY --from=fnetgen /opt/genesis/el /opt/testnet/el
+COPY --from=fnetgen /opt/genesis/build/el /opt/testnet/build/el
 
 RUN cd /opt/testnet && make genesis-cl
 

app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env

@@ -44,7 +44,7 @@ EXECUTION_ENDPOINT=${EXECUTION_ENDPOINT:-http://localhost:8551}
 ETH1_GENESIS_JSON=${ETH1_GENESIS_JSON:-"../build/el/geth.json"}
 ETH1_GENESIS_BLOCK_JSON=${ETH1_GENESIS_BLOCK_JSON:-"../build/el/genesis_block.json"}
 ETH1_CONFIG_YAML=${ETH1_CONFIG_YAML:-"../el/el-config.yaml"}
-ETH1_BLOCK_HASH=${ETH1_BLOCK_HASH:-`cat $ETH1_GENESIS_BLOCK_JSON | jq -r '.result.hash' | cut -d'x' -f2`}
+ETH1_BLOCK_HASH=${ETH1_BLOCK_HASH:-`cat $ETH1_GENESIS_BLOCK_JSON | jq -r '.hash' | cut -d'x' -f2`}
 
 ETH1_CHAIN_ID=${ETH1_CHAIN_ID:-`cat $ETH1_GENESIS_JSON | jq -r '.config.chainId'`}
 ETH1_TTD=${ETH1_TTD:-`cat $ETH1_GENESIS_JSON | jq -r '.config.terminalTotalDifficulty'`}

app/data/container-build/cerc-fixturenet-plugeth-lighthouse/Dockerfile

@@ -1,34 +0,0 @@
-FROM cerc/lighthouse-cli:local AS lcli
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
-FROM cerc/fixturenet-plugeth-plugeth:local AS fnetgeth
-
-FROM cerc/lighthouse:local
-
-# cerc/lighthouse is based on Ubuntu
-RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
-  libssl-dev ca-certificates \
-  curl socat iproute2 telnet wget jq \
-  build-essential python3 python3-dev python3-pip gettext-base \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/*
-
-COPY genesis /opt/testnet
-COPY run-cl.sh /opt/testnet/run.sh
-
-COPY --from=lcli /usr/local/bin/lcli /usr/local/bin/lcli
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/eth2-testnet-genesis
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/eth2-val-tools
-COPY --from=ethgen /apps /apps
-COPY --from=fnetgeth /opt/testnet/el /opt/testnet/el
-COPY --from=fnetgeth /opt/testnet/build/el /opt/testnet/build/el
-
-RUN cd /opt/testnet && make genesis-cl
-
-# Work around some bugs in lcli where the default path is always used.
-RUN mkdir -p /root/.lighthouse && cd /root/.lighthouse && ln -s /opt/testnet/build/cl/testnet
-
-RUN mkdir -p /scripts
-COPY scripts/status-internal.sh /scripts
-COPY scripts/status.sh /scripts
-
-ENTRYPOINT ["/opt/testnet/run.sh"]

app/data/container-build/cerc-fixturenet-plugeth-lighthouse/build.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-# Build cerc/fixturenet-plugeth-lighthouse
-
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-if [ ! -d "${SCRIPT_DIR}/genesis" ]; then
-  cp -frp ${SCRIPT_DIR}/../cerc-fixturenet-eth-lighthouse/genesis ${SCRIPT_DIR}/genesis
-fi
-
-if [ ! -e "${SCRIPT_DIR}/run-cl.sh" ]; then
-  cp -fp ${SCRIPT_DIR}/../cerc-fixturenet-eth-lighthouse/run-cl.sh ${SCRIPT_DIR}/
-fi
-
-if [ ! -d "${SCRIPT_DIR}/scripts" ]; then
-  cp -frp ${SCRIPT_DIR}/../cerc-fixturenet-eth-lighthouse/scripts ${SCRIPT_DIR}/
-fi
-
-docker build -t cerc/fixturenet-plugeth-lighthouse:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR

app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile

@@ -1,42 +1,25 @@
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
+FROM cerc/fixturenet-eth-genesis:local as fnetgen
 
-FROM golang:1.19-alpine as delve
+# Using the same golang image as used to build geth: https://github.com/cerc-io/go-ethereum/blob/HEAD/Dockerfile
+FROM golang:1.20-alpine as delve
 RUN go install github.com/go-delve/delve/cmd/dlv@latest
 
-FROM cerc/plugeth:local as geth
 FROM cerc/plugeth-statediff:local as statediff
+# FIXME: fork of plugeth, use stock after upstreaming patches
+FROM cerc/plugeth:local as geth
 
 FROM alpine:3.17
-RUN apk add --no-cache python3 python3-dev py3-pip curl wget jq build-base gettext libintl openssl bash bind-tools postgresql-client
 
-COPY --from=delve  /go/bin/dlv /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
-COPY --from=ethgen /apps /apps
+RUN apk add --no-cache bash wget python3 bind-tools postgresql-client
 
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
-
-COPY genesis /opt/testnet
 COPY run-el.sh /opt/testnet/run.sh
 
-RUN cd /opt/testnet && make genesis-el
-
-COPY --from=geth  /usr/local/bin/geth /usr/local/bin/
-
-RUN mkdir -p /usr/local/lib/plugeth/
+COPY --from=delve /go/bin/dlv /usr/local/bin/
+COPY --from=geth /usr/local/bin/geth /usr/local/bin/
+COPY --from=fnetgen /opt/genesis /opt/testnet
 COPY --from=statediff  /usr/local/lib/statediff.so /usr/local/lib/plugeth/
 
-# Snag the genesis block info.
+# Initialize the geth db with our config
 RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
-RUN cp -rp ~/ethdata ~/tmpeth && \
-    geth --datadir ~/tmpeth init /opt/testnet/build/el/geth.json && \
-    geth --datadir ~/tmpeth --http & \
-    sleep 5 && \
-    curl -q --location 'localhost:8545' \
-        --header 'Content-Type: application/json' \
-        --data '{ "jsonrpc": "2.0", "id": 14, "method": "eth_getBlockByNumber", "params": ["0x0", false] }' \
-        -o /opt/testnet/build/el/genesis_block.json && \
-    killall -9 geth && \
-    rm -rf ~/tmpeth
 
 ENTRYPOINT ["/opt/testnet/run.sh"]

app/data/container-build/cerc-fixturenet-plugeth-plugeth/build.sh

@@ -6,10 +6,6 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-if [ ! -d "${SCRIPT_DIR}/genesis" ]; then
-  cp -frp ${SCRIPT_DIR}/../cerc-fixturenet-eth-geth/genesis ${SCRIPT_DIR}/genesis
-fi
-
 if [ ! -e "${SCRIPT_DIR}/run-el.sh" ]; then
   cp -fp ${SCRIPT_DIR}/../cerc-fixturenet-eth-geth/run-el.sh ${SCRIPT_DIR}/
 fi

app/data/container-build/cerc-mobymask-ui/Dockerfile

@@ -6,9 +6,6 @@ FROM node:${VARIANT}
 ARG USERNAME=node
 ARG NPM_GLOBAL=/usr/local/share/npm-global
 
-# This container pulls npm package from a registry configured via env var
-ARG CERC_NPM_REGISTRY_URL
-
 # Add NPM global to PATH.
 ENV PATH=${NPM_GLOBAL}/bin:${PATH}
 # Prevents npm from printing version warnings
@@ -33,28 +30,14 @@ RUN \
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     && apt-get -y install --no-install-recommends jq bash netcat
 
-# We do this to get a yq binary from the published container, for the correct architecture we're building here
-COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
-
 RUN mkdir -p /scripts
-COPY ./apply-webapp-config.sh /scripts
-COPY ./start-serving-app.sh /scripts
-
-# Configure the local npm registry
-RUN npm config set @cerc-io:registry ${CERC_NPM_REGISTRY_URL}
-
-RUN mkdir -p /config
 
 # Install simple web server for now (use nginx perhaps later)
 RUN yarn global add http-server
 
-# Globally install both versions of the payload web app package
-# Install old version of MobyMask web app
-RUN yarn global add @cerc-io/mobymask-ui@0.1.4
-# Install the LXDAO version of MobyMask web app
-RUN yarn global add @cerc-io/mobymask-ui-lxdao@npm:@cerc-io/mobymask-ui@0.1.4-lxdao-0.1.1
+WORKDIR /app
+COPY . .
+RUN npm install
 
 # Expose port for http
 EXPOSE 80
-
-CMD ["/scripts/start-serving-app.sh"]

app/data/container-build/cerc-mobymask-ui/apply-webapp-config.sh

@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-    set -x
-fi
-if [[ $# -ne 3 ]]; then
-    echo "Illegal number of parameters" >&2
-    exit 1
-fi
-config_file_name=$1
-webapp_files_dir=$2
-config_prefix=$3
-if ! [[ -f ${config_file_name} ]]; then
-    echo "Config file ${config_file_name} does not exist" >&2
-    exit 1
-fi
-if ! [[ -d ${webapp_files_dir} ]]; then
-    echo "Webapp directory ${webapp_files_dir} does not exist" >&2
-    exit 1
-fi
-# First some magic using sed to translate our yaml config file into an array of key value pairs like:
-# ${config_prefix}<path-through-objects>=<value>
-# sed "s/'//g" is used to remove single quote for relayNodes value
-readarray -t config_kv_pair_array < <( sed -E 's/([^:]+):\s*(.*)/\1=\2/g' ${config_file_name} | sed "s/'//g" |  sed "s/^/${config_prefix}_/" )
-declare -p config_kv_pair_array
-# Then iterate over that kv array making the template substitution in our web app files
-for kv_pair_string in "${config_kv_pair_array[@]}"
-do
-    kv_pair=(${kv_pair_string//=/ })
-    template_string_to_replace=${kv_pair[0]}
-    template_value_to_substitute=${kv_pair[1]}
-    # Run find and sed to do the substitution of one variable over all files
-    # See: https://stackoverflow.com/a/21479607/1701505
-    echo "Substituting: ${template_string_to_replace} = ${template_value_to_substitute}"
-
-    # TODO: Pass keys to be replaced without double quotes
-    if [[ "$template_string_to_replace" =~ ^${config_prefix}_(relayNodes|chainId|denyMultiaddrs)$ ]]; then
-        find ${webapp_files_dir} -type f -exec sed -i 's#"'"${template_string_to_replace}"'"#'"${template_value_to_substitute}"'#g' {} +
-    else
-        # Note: we do not escape our strings, on the expectation they do not container the '#' char.
-        find ${webapp_files_dir} -type f -exec sed -i 's#'${template_string_to_replace}'#'${template_value_to_substitute}'#g' {} +
-    fi
-done

app/data/container-build/cerc-mobymask-ui/build.sh

@@ -8,5 +8,4 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 CERC_NPM_REGISTRY_URL="https://git.vdb.to/api/packages/cerc-io/npm/"
 
-docker build -t cerc/mobymask-ui:local  ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile \
-  --build-arg CERC_NPM_REGISTRY_URL ${SCRIPT_DIR}
+docker build -t cerc/mobymask-ui:local  ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/mobymask-ui

app/data/container-build/cerc-mobymask-ui/start-serving-app.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-    set -x
-fi
-
-# TODO: Don't hard wire this:
-webapp_files_dir="/usr/local/share/.config/yarn/global/node_modules/${CERC_BUILD_DIR}"
-/scripts/apply-webapp-config.sh /config/config.yml ${webapp_files_dir} MOBYMASK_HOSTED_CONFIG
-http-server -p 80 ${webapp_files_dir}

app/data/container-build/cerc-plugeth-statediff/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build cerc/plugeth-statediff
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/plugeth-statediff:local ${build_command_args} ${CERC_REPO_BASE_DIR}/plugeth-statediff

app/data/container-build/cerc-plugeth/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build cerc/plugeth
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/plugeth:local ${build_command_args} ${CERC_REPO_BASE_DIR}/plugeth

app/data/repository-list.txt

@@ -37,3 +37,5 @@ git.vdb.to/cerc-io/test-project
 github.com/Fantom-foundation/go-opera
 github.com/cerc-io/lasso
 github.com/paradigmxyz/reth
+git.vdb.to/cerc-io/plugeth
+git.vdb.to/cerc-io/plugeth-statediff

app/data/stacks/fixturenet-eth-loaded/stack.yml

@@ -11,6 +11,7 @@ containers:
   - cerc/go-ethereum
   - cerc/lighthouse
   - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-genesis
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/ipld-eth-server

app/data/stacks/fixturenet-eth-tx/stack.yml

@@ -10,6 +10,7 @@ containers:
   - cerc/go-ethereum
   - cerc/lighthouse
   - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-genesis
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/tx-spammer

app/data/stacks/fixturenet-eth/stack.yml

@@ -9,6 +9,7 @@ containers:
   - cerc/go-ethereum
   - cerc/lighthouse
   - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-genesis
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/foundry

app/data/stacks/fixturenet-optimism/stack.yml

@@ -11,6 +11,7 @@ containers:
   - cerc/go-ethereum
   - cerc/lighthouse
   - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-genesis
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/foundry

app/data/stacks/fixturenet-plugeth-tx/stack.yml

@@ -2,27 +2,28 @@ version: "1.2"
 name: fixturenet-plugeth-tx
 decription: "plugeth Ethereum Fixturenet w/ tx-spammer"
 repos:
-  - github.com/cerc-io/tx-spammer
-  - github.com/dboreham/foundry
+  - git.vdb.to/cerc-io/plugeth@statediff-wip
+  - git.vdb.to/cerc-io/plugeth-statediff
   - github.com/cerc-io/lighthouse
   - github.com/cerc-io/ipld-eth-db@v5
   - github.com/cerc-io/ipld-eth-server@v5
-  - git.vdb.to/cerc-io/plugeth@statediff-wip
-  - git.vdb.to/cerc-io/plugeth-statediff@dev-local-build
+  - github.com/cerc-io/tx-spammer
+  - github.com/dboreham/foundry
 containers:
-  - cerc/lighthouse
-  - cerc/lighthouse-cli
   - cerc/plugeth-statediff
   - cerc/plugeth
+  - cerc/fixturenet-eth-genesis
   - cerc/fixturenet-plugeth-plugeth
-  - cerc/fixturenet-plugeth-lighthouse
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-lighthouse
   - cerc/tx-spammer
   - cerc/foundry
   - cerc/ipld-eth-db
   - cerc/ipld-eth-server
 pods:
+  - fixturenet-plugeth
   - ipld-eth-db
   - ipld-eth-server
-  - fixturenet-plugeth
-  - foundry
   - tx-spammer
+  - foundry

app/data/stacks/fixturenet-pocket/stack.yml

@@ -10,6 +10,7 @@ containers:
   - cerc/go-ethereum
   - cerc/lighthouse
   - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-genesis
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
   - cerc/pocket

app/data/stacks/mobymask-v2/stack.yml

@@ -7,8 +7,9 @@ repos:
   - github.com/ethereum-optimism/optimism@v1.0.4
   - github.com/ethereum-optimism/op-geth@v1.101105.2
   - github.com/cerc-io/watcher-ts@v0.2.43
-  - github.com/cerc-io/mobymask-v2-watcher-ts@v0.1.1
+  - github.com/cerc-io/mobymask-v2-watcher-ts@v0.1.2
   - github.com/cerc-io/MobyMask@v0.1.2
+  - github.com/cerc-io/mobymask-ui
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse

app/data/stacks/test/deploy/commands.py

@@ -45,6 +45,13 @@ def init(command_context: DeployCommandContext):
     return yaml.load(default_spec_file_content)
 
 
+def create(command_context: DeployCommandContext):
+    data = "create-command-output-data"
+    output_file_path = command_context.deployment_dir.joinpath("create-file")
+    with open(output_file_path, 'w+') as output_file:
+        output_file.write(data)
+
+
 def get_state(command_context: DeployCommandContext):
     print("Here we get state")
     return State.CONFIGURED

app/deploy.py

@@ -144,14 +144,16 @@ def exec_operation(ctx, extra_args):
             print(f"container command returned error exit status")
 
 
-def logs_operation(ctx, extra_args):
+def logs_operation(ctx, tail: int, follow: bool, extra_args: str):
     global_context = ctx.parent.parent.obj
     extra_args_list = list(extra_args) or None
     if not global_context.dry_run:
         if global_context.verbose:
             print("Running compose logs")
-        logs_output = ctx.obj.docker.compose.logs(services=extra_args_list if extra_args_list is not None else [])
-        print(logs_output)
+        services_list = extra_args_list if extra_args_list is not None else []
+        logs_stream = ctx.obj.docker.compose.logs(services=services_list, tail=tail, follow=follow, stream=True)
+        for stream_type, stream_content in logs_stream:
+            print(stream_content.decode("utf-8"), end="")
 
 
 @command.command()
@@ -192,10 +194,12 @@ def exec(ctx, extra_args):
 
 
 @command.command()
+@click.option("--tail", "-n", default=None, help="number of lines to display")
+@click.option("--follow", "-f", is_flag=True, default=False, help="follow log output")
 @click.argument('extra_args', nargs=-1)  # help: command: logs <service1> <service2>
 @click.pass_context
-def logs(ctx, extra_args):
-    logs_operation(ctx, extra_args)
+def logs(ctx, tail, follow, extra_args):
+    logs_operation(ctx, tail, follow, extra_args)
 
 
 def get_stack_status(ctx, stack):

app/deployment.py

@@ -119,11 +119,13 @@ def exec(ctx, extra_args):
 
 
 @command.command()
+@click.option("--tail", "-n", default=None, help="number of lines to display")
+@click.option("--follow", "-f", is_flag=True, default=False, help="follow log output")
 @click.argument('extra_args', nargs=-1)  # help: command: logs <service1> <service2>
 @click.pass_context
-def logs(ctx, extra_args):
+def logs(ctx, tail, follow, extra_args):
     ctx.obj = make_deploy_context(ctx)
-    logs_operation(ctx, extra_args)
+    logs_operation(ctx, tail, follow, extra_args)
 
 
 @command.command()

app/deployment_create.py

@@ -205,7 +205,11 @@ def create(ctx, spec_file, deployment_dir):
                 if not os.path.exists(destination_config_dir):
                     copytree(source_config_dir, destination_config_dir)
     # Delegate to the stack's Python code
-    deployment_context = DeploymentContext(Path(deployment_dir), ctx.obj)
+    # The deploy create command doesn't require a --stack argument so we need to insert the
+    # stack member here.
+    deployment_command_context = ctx.obj
+    deployment_command_context.stack = stack_name
+    deployment_context = DeploymentContext(Path(deployment_dir), deployment_command_context)
     call_stack_deploy_create(deployment_context)
 
 

requirements.txt

@@ -1,7 +1,8 @@
-python-decouple>=3.6
-GitPython>=3.1.27
-tqdm>=4.64.0
-python-on-whales>=0.58.0
-click>=8.1.3
-pyyaml>=6.0
+python-decouple>=3.8
+GitPython>=3.1.32
+tqdm>=4.65.0
+python-on-whales>=0.64.0
+click>=8.1.6
+PyYAML>=6.0.1
 ruamel.yaml>=0.17.32
+pydantic==1.10.9

tests/deploy/run-deploy-test.sh

@@ -26,17 +26,18 @@ $TEST_TARGET_SO --stack test build-containers
 # Test deploy command execution
 $TEST_TARGET_SO --stack test deploy setup $CERC_REPO_BASE_DIR
 # Check that we now have the expected output directory
-if [ ! -d "$CERC_REPO_BASE_DIR/container-output-dir" ]; then
+container_output_dir=$CERC_REPO_BASE_DIR/container-output-dir
+if [ ! -d "$container_output_dir" ]; then
     echo "deploy setup test: output directory not present"
     echo "deploy setup test: FAILED"
     exit 1
 fi
-if [ ! -f "$CERC_REPO_BASE_DIR/container-output-dir/output-file" ]; then
+if [ ! -f "$container_output_dir/output-file" ]; then
     echo "deploy setup test: output file not present"
     echo "deploy setup test: FAILED"
     exit 1
 fi
-output_file_content=$(<$CERC_REPO_BASE_DIR/container-output-dir/output-file)
+output_file_content=$(<$container_output_dir/output-file)
 if [ ! "$output_file_content" == "output-data"  ]; then
     echo "deploy setup test: output file contents not correct"
     echo "deploy setup test: FAILED"
@@ -73,4 +74,49 @@ else
     exit 1
 fi
 $TEST_TARGET_SO --stack test deploy down --delete-volumes
+# Basic test of creating a deployment
+test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
+test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
+$TEST_TARGET_SO --stack test deploy init --output $test_deployment_spec
+# Check the file now exists
+if [ ! -f "$test_deployment_spec" ]; then
+    echo "deploy init test: spec file not present"
+    echo "deploy init test: FAILED"
+    exit 1
+fi
+echo "deploy init test: passed"
+$TEST_TARGET_SO deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+# Check the deployment dir exists
+if [ ! -d "$test_deployment_dir" ]; then
+    echo "deploy create test: deployment directory not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create test: passed"
+# Check the file writted by the create command in the stack now exists
+if [ ! -f "$test_deployment_dir/create-file" ]; then
+    echo "deploy create test: create output file not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+# And has the right content
+create_file_content=$(<$test_deployment_dir/create-file)
+if [ ! "$create_file_content" == "create-command-output-data"  ]; then
+    echo "deploy create test: create output file contents not correct"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create output file test: passed"
+# Try to start the deployment
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+# Check logs command works
+log_output_2=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_2" == *"Filesystem is fresh"* ]]; then
+    echo "deployment logs test: passed"
+else
+    echo "deployment logs test: FAILED"
+    exit 1
+fi
+# Stop and clean up
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
 echo "Test passed"