Catch and report git errors (#353)

Former-commit-id: 7c867171e4

.gitignore

@@ -5,4 +5,5 @@ laconic-so
 laconic_stack_orchestrator.egg-info
 __pycache__
 *~
-build
+package
+app/data/build_tag.txt

app/data/compose/docker-compose-fixturenet-eth.yml

@@ -8,6 +8,8 @@ services:
     environment:
       RUN_BOOTNODE: "true"
     image: cerc/fixturenet-eth-geth:local
+    volumes:
+      - fixturenet_eth_bootnode_geth_data:/root/ethdata
     ports:
       - "9898"
       - "30303"
@@ -26,6 +28,7 @@ services:
     image: cerc/fixturenet-eth-geth:local
     volumes:
       - fixturenet_geth_accounts:/opt/testnet/build/el
+      - fixturenet_eth_geth_1_data:/root/ethdata
     healthcheck:
       test: ["CMD", "nc", "-v", "localhost", "8545"]
       interval: 30s
@@ -52,12 +55,16 @@ services:
     image: cerc/fixturenet-eth-geth:local
     depends_on:
       - fixturenet-eth-bootnode-geth
+    volumes:
+      - fixturenet_eth_geth_2_data:/root/ethdata
 
   fixturenet-eth-bootnode-lighthouse:
     hostname: fixturenet-eth-bootnode-lighthouse
     environment:
       RUN_BOOTNODE: "true"
     image: cerc/fixturenet-eth-lighthouse:local
+    volumes:
+      - fixturenet_eth_bootnode_lighthouse_data:/opt/testnet/build/cl
 
   fixturenet-eth-lighthouse-1:
     hostname: fixturenet-eth-lighthouse-1
@@ -74,6 +81,8 @@ services:
       ETH1_ENDPOINT: "http://fixturenet-eth-geth-1:8545"
       EXECUTION_ENDPOINT: "http://fixturenet-eth-geth-1:8551"
     image: cerc/fixturenet-eth-lighthouse:local
+    volumes:
+      - fixturenet_eth_lighthouse_1_data:/opt/testnet/build/cl
     depends_on:
       fixturenet-eth-bootnode-lighthouse:
         condition: service_started
@@ -98,6 +107,8 @@ services:
       EXECUTION_ENDPOINT: "http://fixturenet-eth-geth-2:8551"
       LIGHTHOUSE_GENESIS_STATE_URL: "http://fixturenet-eth-lighthouse-1:8001/eth/v2/debug/beacon/states/0"
     image: cerc/fixturenet-eth-lighthouse:local
+    volumes:
+      - fixturenet_eth_lighthouse_2_data:/opt/testnet/build/cl
     depends_on:
       fixturenet-eth-bootnode-lighthouse:
         condition: service_started
@@ -106,3 +117,9 @@ services:
 
 volumes:
   fixturenet_geth_accounts:
+  fixturenet_eth_bootnode_geth_data:
+  fixturenet_eth_geth_1_data:
+  fixturenet_eth_geth_2_data:
+  fixturenet_eth_bootnode_lighthouse_data:
+  fixturenet_eth_lighthouse_1_data:
+  fixturenet_eth_lighthouse_2_data:

app/data/compose/docker-compose-fixturenet-laconic-console.yml

@@ -2,5 +2,7 @@ services:
   laconic-console:
     restart: unless-stopped
     image: cerc/laconic-console-host:local
+    environment:
+      - LACONIC_HOSTED_ENDPOINT=${LACONIC_HOSTED_ENDPOINT:-http://localhost}
     ports:
       - "80"

app/data/compose/docker-compose-mobymask-app.yml

@@ -3,6 +3,7 @@ version: '3.2'
 services:
   # Builds and serves the MobyMask react-app
   mobymask-app:
+    restart: unless-stopped
     image: cerc/mobymask-ui:local
     env_file:
       - ../config/watcher-mobymask-v2/mobymask-params.env
@@ -12,6 +13,7 @@ services:
       CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
+    working_dir: /scripts
     # Waits for watcher server to be up before app build
     # Required when running with watcher stack to get deployed contract address
     command:
@@ -19,15 +21,14 @@ services:
       - -c
       - ./wait-for-it.sh -h ${CERC_WATCHER_HOST:-$${DEFAULT_CERC_WATCHER_HOST}} -p ${CERC_WATCHER_PORT:-$${DEFAULT_CERC_WATCHER_PORT}} -s -t 0 -- ./mobymask-app-start.sh
     volumes:
-      - ../config/wait-for-it.sh:/app/wait-for-it.sh
-      - ../config/watcher-mobymask-v2/mobymask-app-config.json:/app/src/mobymask-app-config.json
-      - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/app/mobymask-app-start.sh
+      - ../config/wait-for-it.sh:/scripts/wait-for-it.sh
+      - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
       - peers_ids:/peers
       - mobymask_deployment:/server
     ports:
-      - "0.0.0.0:3002:3000"
+      - "0.0.0.0:3002:80"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3000"]
+      test: ["CMD", "nc", "-v", "localhost", "80"]
       interval: 20s
       timeout: 5s
       retries: 15

app/data/compose/docker-compose-peer-test-app.yml

@@ -4,21 +4,24 @@ services:
   peer-test-app:
     # Builds and serves the peer-test react-app
     image: cerc/react-peer:local
-    working_dir: /app/packages/test-app
+    working_dir: /scripts
     env_file:
       - ../config/watcher-mobymask-v2/mobymask-params.env
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
-    command: ["sh", "./test-app-start.sh"]
+    command:
+      - sh
+      - -c
+      - ./wait-for-it.sh -h ${CERC_WATCHER_HOST:-$${DEFAULT_CERC_WATCHER_HOST}} -p ${CERC_WATCHER_PORT:-$${DEFAULT_CERC_WATCHER_PORT}} -s -t 0 -- ./test-app-start.sh
     volumes:
-      - ../config/watcher-mobymask-v2/test-app-config.json:/app/packages/test-app/src/test-app-config.json
-      - ../config/watcher-mobymask-v2/test-app-start.sh:/app/packages/test-app/test-app-start.sh
+      - ../config/wait-for-it.sh:/scripts/wait-for-it.sh
+      - ../config/watcher-mobymask-v2/test-app-start.sh:/scripts/test-app-start.sh
       - peers_ids:/peers
     ports:
-      - "0.0.0.0:3003:3000"
+      - "0.0.0.0:3003:80"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3000"]
+      test: ["CMD", "nc", "-v", "localhost", "80"]
       interval: 20s
       timeout: 5s
       retries: 15

app/data/compose/docker-compose-watcher-mobymask-v2.yml

@@ -54,6 +54,18 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Creates peer-id files if they don't exist
+  peer-ids-gen:
+    image: cerc/watcher-ts:local
+    restart: on-failure
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+    working_dir: /app/packages/peer
+    command: ["sh", "generate-peer-ids.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v2/generate-peer-ids.sh:/app/packages/peer/generate-peer-ids.sh
+      - peers_ids:/peer-ids
+
   # Starts the mobymask-v2-watcher server
   mobymask-watcher-server:
     image: cerc/watcher-mobymask-v2:local
@@ -61,6 +73,8 @@ services:
     depends_on:
       mobymask-watcher-db:
         condition: service_healthy
+      peer-ids-gen:
+        condition: service_completed_successfully
       mobymask:
         condition: service_completed_successfully
     env_file:
@@ -76,9 +90,9 @@ services:
       CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
     command: ["sh", "start-server.sh"]
     volumes:
-      - ../config/watcher-mobymask-v2/watcher-config-template.toml:/app/packages/mobymask-v2-watcher/environments/watcher-config-template.toml
-      - ../config/watcher-mobymask-v2/start-server.sh:/app/packages/mobymask-v2-watcher/start-server.sh
-      - peers_ids:/app/packages/mobymask-v2-watcher/peers
+      - ../config/watcher-mobymask-v2/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-mobymask-v2/start-server.sh:/app/start-server.sh
+      - peers_ids:/app/peers
       - mobymask_deployment:/server
       - fixturenet_geth_accounts:/geth-accounts:ro
     # Expose GQL, metrics and relay node ports
@@ -95,6 +109,28 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Container to run peer tests
+  peer-tests:
+    image: cerc/watcher-ts:local
+    restart: on-failure
+    depends_on:
+      mobymask-watcher-server:
+        condition: service_healthy
+      peer-ids-gen:
+        condition: service_completed_successfully
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+    working_dir: /app/packages/peer
+    command:
+      - sh
+      - -c
+      - |
+        ./set-tests-env.sh && \
+        tail -f /dev/null
+    volumes:
+      - ../config/watcher-mobymask-v2/set-tests-env.sh:/app/packages/peer/set-tests-env.sh
+      - peers_ids:/peer-ids
+
 volumes:
   mobymask_watcher_db_data:
   peers_ids:

app/data/config/fixturenet-optimism/run-op-batcher.sh

@@ -9,6 +9,15 @@ CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
 # Get BACTHER_KEY from keys.json
 BATCHER_KEY=$(jq -r '.Batcher.privateKey' /l2-accounts/keys.json | tr -d '"')
 
+cleanup() {
+    echo "Signal received, cleaning up..."
+    kill ${batcher_pid}
+
+    wait
+    echo "Done"
+}
+trap 'cleanup' INT TERM
+
 op-batcher \
   --l2-eth-rpc=http://op-geth:8545 \
   --rollup-rpc=http://op-node:8547 \
@@ -23,4 +32,8 @@ op-batcher \
   --max-channel-duration=1 \
   --target-l1-tx-size-bytes=2048 \
   --l1-eth-rpc=$CERC_L1_RPC \
-  --private-key=$BATCHER_KEY
+  --private-key=$BATCHER_KEY \
+  &
+
+batcher_pid=$!
+wait $batcher_pid

app/data/config/fixturenet-optimism/run-op-geth.sh

@@ -47,6 +47,15 @@ fi
 SEQUENCER_ADDRESS=$(jq -r '.Sequencer.address' /l2-accounts/keys.json | tr -d '"')
 echo "SEQUENCER_ADDRESS: ${SEQUENCER_ADDRESS}"
 
+cleanup() {
+    echo "Signal received, cleaning up..."
+    kill ${geth_pid}
+
+    wait
+    echo "Done"
+}
+trap 'cleanup' INT TERM
+
 # Run op-geth
 geth \
   --datadir ./datadir \
@@ -74,4 +83,8 @@ geth \
   --allow-insecure-unlock \
   --mine \
   --miner.etherbase=$SEQUENCER_ADDRESS \
-  --unlock=$SEQUENCER_ADDRESS
+  --unlock=$SEQUENCER_ADDRESS \
+  &
+
+geth_pid=$!
+wait $geth_pid

app/data/config/watcher-mobymask-v2/generate-peer-ids.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Check for peer ids in ./peers folder, create if not present
+if [ -f /peer-ids/relay-id.json ]; then
+  echo "Using peer id for relay node from the mounted volume"
+else
+  echo "Creating a new peer id for relay node"
+  yarn create-peer -f /peer-ids/relay-id.json
+fi
+
+if [ -f /peer-ids/peer-id.json ]; then
+  echo "Using peer id for peer node from the mounted volume"
+else
+  echo "Creating a new peer id for peer node"
+  yarn create-peer -f /peer-ids/peer-id.json
+fi

app/data/config/watcher-mobymask-v2/mobymask-app-config.json

@@ -1,7 +0,0 @@
-{
-  "name": "MobyMask",
-  "relayNodes": [],
-  "peer": {
-    "enableDebugInfo": true
-  }
-}

app/data/config/watcher-mobymask-v2/mobymask-app-start.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 set -e
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
@@ -28,13 +28,10 @@ else
   echo "Taking deployed contract details from env"
 fi
 
-# Export config values in a json file
-jq --arg address "$CERC_DEPLOYED_CONTRACT" \
-  --argjson chainId "$CERC_CHAIN_ID" \
-  --argjson relayNodes "$CERC_RELAY_NODES" \
-  '.address = $address | .chainId = $chainId | .relayNodes = $relayNodes' \
-  /app/src/mobymask-app-config.json > /app/src/config.json
+# Use yq to create config.yml with environment variables
+yq -n ".address = env(CERC_DEPLOYED_CONTRACT)" > /config/config.yml
+yq ".watcherUrl = env(CERC_APP_WATCHER_URL)" -i /config/config.yml
+yq ".chainId = env(CERC_CHAIN_ID)" -i /config/config.yml
+yq ".relayNodes = strenv(CERC_RELAY_NODES)" -i /config/config.yml
 
-REACT_APP_WATCHER_URI="$CERC_APP_WATCHER_URL/graphql" npm run build
-
-serve -s build
+/scripts/start-serving-app.sh

app/data/config/watcher-mobymask-v2/set-tests-env.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+CERC_RELAY_MULTIADDR="/dns4/mobymask-watcher-server/tcp/9090/ws/p2p/$(jq -r '.id' /peer-ids/relay-id.json)"
+
+# Write the relay node's multiaddr to /app/packages/peer/.env for running tests
+echo "RELAY=\"$CERC_RELAY_MULTIADDR\"" > ./.env

app/data/config/watcher-mobymask-v2/start-server.sh

@@ -14,26 +14,7 @@ CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRA
 
 echo "Using L2 RPC endpoint ${CERC_L2_GETH_RPC}"
 
-# Check for peer ids in ./peers folder, create if not present
-if [ -f ./peers/relay-id.json ]; then
-  echo "Using peer id for relay node from the mounted volume"
-else
-  echo "Creating a new peer id for relay node"
-  cd ../peer
-  yarn create-peer -f ../mobymask-v2-watcher/peers/relay-id.json
-  cd ../mobymask-v2-watcher
-fi
-
-if [ -f ./peers/peer-id.json ]; then
-  echo "Using peer id for peer node from the mounted volume"
-else
-  echo "Creating a new peer id for peer node"
-  cd ../peer
-  yarn create-peer -f ../mobymask-v2-watcher/peers/peer-id.json
-  cd ../mobymask-v2-watcher
-fi
-
-CERC_RELAY_MULTIADDR="/dns4/mobymask-watcher-server/tcp/9090/ws/p2p/$(jq -r '.id' ./peers/relay-id.json)"
+CERC_RELAY_MULTIADDR="/dns4/mobymask-watcher-server/tcp/9090/ws/p2p/$(jq -r '.id' /app/peers/relay-id.json)"
 
 # Use contract address from environment variable or set from config.json in mounted volume
 if [ -n "$CERC_DEPLOYED_CONTRACT" ]; then
@@ -65,8 +46,5 @@ WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
 
-# Write the relay node's multiaddr to /app/packages/peer/.env for running tests
-echo "RELAY=\"$CERC_RELAY_MULTIADDR\"" > /app/packages/peer/.env
-
 echo 'yarn server'
 yarn server

app/data/config/watcher-mobymask-v2/test-app-start.sh

@@ -14,11 +14,7 @@ fi
 
 echo "Using CERC_RELAY_NODES $CERC_RELAY_NODES"
 
-# Set relay nodes in config using CERC_RELAY_NODES
-jq --argjson relayNodes "$CERC_RELAY_NODES" \
-  '.relayNodes = $relayNodes' \
-  ./src/test-app-config.json > ./src/config.json
+# Use yq to create config.yml with environment variables
+yq -n ".relayNodes = strenv(CERC_RELAY_NODES)" > /config/config.yml
 
-yarn build
-
-serve -s build
+/scripts/start-serving-app.sh

app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile

@@ -22,6 +22,6 @@ COPY run-el.sh /opt/testnet/run.sh
 RUN cd /opt/testnet && make genesis-el
 
 COPY --from=geth  /usr/local/bin/geth /usr/local/bin/
-RUN geth init /opt/testnet/build/el/geth.json && rm -f ~/.ethereum/geth/nodekey
+RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
 
 ENTRYPOINT ["/opt/testnet/run.sh"]

app/data/container-build/cerc-fixturenet-eth-geth/genesis/accounts/import_keys.sh

@@ -12,6 +12,6 @@ for line in `cat ../build/el/accounts.csv`; do
 
   echo ""
   echo "$ADDRESS"
-  geth account import --password .pw.$$ .key.$$
+  geth account import --datadir=~/ethdata --password .pw.$$ .key.$$
   rm -f .pw.$$ .key.$$
 done

app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -18,17 +18,35 @@ if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
     START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --"
 fi
 
+# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script
+cleanup() {
+    echo "Signal received, cleaning up..."
+
+    # Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process)
+    pkill -P ${geth_pid}
+    sleep 2
+    kill $(jobs -p)
+
+    wait
+    echo "Done"
+}
+trap 'cleanup' SIGINT SIGTERM
+
 if [ "true" == "$RUN_BOOTNODE" ]; then
     $START_CMD \
+       --datadir=~/ethdata \
       --nodekeyhex="${BOOTNODE_KEY}" \
       --nodiscover \
       --ipcdisable \
       --networkid=${NETWORK_ID} \
-      --netrestrict="${NETRESTRICT}"
+      --netrestrict="${NETRESTRICT}" \
+      &
+
+    geth_pid=$!
 else
     cd /opt/testnet/accounts
     ./import_keys.sh
-    
+
     echo -n "$JWT" > /opt/testnet/build/el/jwtsecret
 
     if [ "$CERC_RUN_STATEDIFF" == "detect" ] && [ -n "$CERC_STATEDIFF_DB_HOST" ]; then
@@ -74,6 +92,7 @@ else
     fi
 
     $START_CMD \
+      --datadir=~/ethdata \
       --bootnodes="${ENODE}" \
       --allow-insecure-unlock \
       --http \
@@ -101,5 +120,10 @@ else
       --metrics.addr="0.0.0.0" \
       --verbosity=${CERC_GETH_VERBOSITY:-3} \
       --vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
-      --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS}
+      --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS} \
+      &
+
+    geth_pid=$!
 fi
+
+wait $geth_pid

app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/beacon_node.sh

@@ -19,9 +19,9 @@ http_port=8001
 authrpc_port=8551
 
 exec lighthouse \
-  --debug-level $DEBUG_LEVEL \
   bn \
   $SUBSCRIBE_ALL_SUBNETS \
+  --debug-level $DEBUG_LEVEL \
   --boot-nodes "$ENR" \
   --datadir $data_dir \
   --testnet-dir $TESTNET_DIR \
@@ -38,4 +38,5 @@ exec lighthouse \
   --execution-jwt $JWTSECRET \
   --terminal-total-difficulty-override $ETH1_TTD \
   --suggested-fee-recipient $SUGGESTED_FEE_RECIPIENT \
-  --target-peers $((BN_COUNT - 1))
+  --target-peers $((BN_COUNT - 1)) \
+  --http-allow-sync-stalled \

app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/bootnode.sh

@@ -21,14 +21,22 @@ if [ ! -f "$DATADIR/bootnode/enr.dat" ]; then
     --udp-port $BOOTNODE_PORT \
     --tcp-port $BOOTNODE_PORT \
     --genesis-fork-version $GENESIS_FORK_VERSION \
-    --output-dir $DATADIR/bootnode
+    --output-dir $DATADIR/bootnode-temp
 
-    bootnode_enr=`cat $DATADIR/bootnode/enr.dat`
-    echo "- $bootnode_enr" > $TESTNET_DIR/boot_enr.yaml
-    
-    echo "Generated bootnode enr and written to $TESTNET_DIR/boot_enr.yaml"
+  # Output ENR to a temp dir and mv as "lcli generate-bootnode-enr" will not overwrite an empty dir (mounted volume)
+  mkdir -p $DATADIR/bootnode
+  mv $DATADIR/bootnode-temp/* $DATADIR/bootnode
+  rm -r $DATADIR/bootnode-temp
+
+  echo "Generated bootnode enr"
+else
+  echo "Found existing bootnode enr"
 fi
 
+bootnode_enr=`cat $DATADIR/bootnode/enr.dat`
+echo "- $bootnode_enr" > $TESTNET_DIR/boot_enr.yaml
+echo "Written bootnode enr to $TESTNET_DIR/boot_enr.yaml"
+
 exec lighthouse boot_node \
     --testnet-dir $TESTNET_DIR \
     --port $BOOTNODE_PORT \

app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/reset_genesis_time.sh

@@ -16,4 +16,4 @@ lcli \
   $TESTNET_DIR/genesis.ssz \
   $NOW
 
-echo "Reset genesis time to now ($NOW)"
+echo "Reset genesis time to ($NOW)"

app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/validator_client.sh

@@ -21,9 +21,9 @@ while getopts "pd:" flag; do
 done
 
 exec lighthouse \
-  --debug-level $DEBUG_LEVEL \
   vc \
   $BUILDER_PROPOSALS \
+  --debug-level $DEBUG_LEVEL \
   --validators-dir $DATADIR/node_$NODE_NUMBER/validators \
   --secrets-dir $DATADIR/node_$NODE_NUMBER/secrets \
   --testnet-dir $TESTNET_DIR \

app/data/container-build/cerc-fixturenet-eth-lighthouse/run-cl.sh

@@ -1,12 +1,25 @@
 #!/bin/bash
 
-if [ "true" == "$RUN_BOOTNODE" ]; then 
+# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script
+cleanup() {
+    echo "Signal received, cleaning up..."
+    kill $(jobs -p)
+
+    wait
+    echo "Done"
+}
+trap 'cleanup' SIGINT SIGTERM
+
+if [ "true" == "$RUN_BOOTNODE" ]; then
     cd /opt/testnet/build/cl
     python3 -m http.server 3000 &
 
 
     cd /opt/testnet/cl
-    ./bootnode.sh 2>&1 | tee /var/log/lighthouse_bootnode.log
+    ./bootnode.sh 2>&1 | tee /var/log/lighthouse_bootnode.log &
+    bootnode_pid=$!
+
+    wait $bootnode_pid
 else
     while [ 1 -eq 1 ]; do
       echo "Waiting on geth ..."
@@ -25,7 +38,12 @@ else
     cd /opt/testnet/cl
 
     if [ -z "$LIGHTHOUSE_GENESIS_STATE_URL" ]; then
-        ./reset_genesis_time.sh
+        # Check if beacon node data exists to avoid resetting genesis time on a restart
+        if [ -d /opt/testnet/build/cl/node_"$NODE_NUMBER"/beacon ]; then
+            echo "Skipping genesis time reset"
+        else
+            ./reset_genesis_time.sh
+        fi
     else
         while [ 1 -eq 1 ]; do
             echo "Waiting on Genesis time ..."
@@ -54,10 +72,9 @@ else
     echo -n "$JWT" > $JWTSECRET
 
     ./beacon_node.sh 2>&1 | tee /var/log/lighthouse_bn.log &
-    lpid=$!
+    beacon_pid=$!
     ./validator_client.sh 2>&1 | tee /var/log/lighthouse_vc.log &
-    vpid=$!
+    validator_pid=$!
 
-    wait $lpid $vpid
+    wait $beacon_pid $validator_pid
 fi
-

app/data/container-build/cerc-laconic-console-host/apply-webapp-config.sh

@@ -26,9 +26,10 @@ do
     kv_pair=(${kv_pair_string//=/ })
     template_string_to_replace=${kv_pair[0]}
     template_value_to_substitute=${kv_pair[1]}
+    template_value_to_substitute_expanded=${template_value_to_substitute//LACONIC_HOSTED_ENDPOINT/${LACONIC_HOSTED_ENDPOINT}}
     # Run find and sed to do the substitution of one variable over all files
     # See: https://stackoverflow.com/a/21479607/1701505
-    echo "Substituting: ${template_string_to_replace} = ${template_value_to_substitute}"
+    echo "Substituting: ${template_string_to_replace} = ${template_value_to_substitute_expanded}"
     # Note: we do not escape our strings, on the expectation they do not container the '#' char.
-    find ${webapp_files_dir} -type f -exec sed -i 's#'${template_string_to_replace}'#'${template_value_to_substitute}'#g' {} +
+    find ${webapp_files_dir} -type f -exec sed -i 's#'${template_string_to_replace}'#'${template_value_to_substitute_expanded}'#g' {} +
 done

app/data/container-build/cerc-laconic-console-host/config.yml

@@ -2,5 +2,5 @@
 
 services:
   wns:
-    server: 'http://localhost:9473/api'
-    webui: 'http://localhost:9473/console'
+    server: 'LACONIC_HOSTED_ENDPOINT:9473/api'
+    webui: 'LACONIC_HOSTED_ENDPOINT:9473/console'

app/data/container-build/cerc-mobymask-ui/Dockerfile

@@ -1,14 +1,57 @@
-FROM node:18.15.0-alpine3.16
+# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT=16-bullseye
+FROM node:${VARIANT}
 
-RUN apk --update --no-cache add make git jq bash
+ARG USERNAME=node
+ARG NPM_GLOBAL=/usr/local/share/npm-global
 
-WORKDIR /app
+# This container pulls npm package from a registry configured via env var
+ARG CERC_NPM_REGISTRY_URL
 
-COPY . .
+# Add NPM global to PATH.
+ENV PATH=${NPM_GLOBAL}/bin:${PATH}
+# Prevents npm from printing version warnings
+ENV NPM_CONFIG_UPDATE_NOTIFIER=false
 
-RUN npm install -g serve
+RUN \
+  # Configure global npm install location, use group to adapt to UID/GID changes
+  if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \
+  && usermod -a -G npm ${USERNAME} \
+  && umask 0002 \
+  && mkdir -p ${NPM_GLOBAL} \
+  && touch /usr/local/etc/npmrc \
+  && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \
+  && chmod g+s ${NPM_GLOBAL} \
+  && npm config -g set prefix ${NPM_GLOBAL} \
+  && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
+  # Install eslint
+  && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+  && npm cache clean --force > /dev/null 2>&1
 
-RUN echo "Building mobymask-ui" && \
-    npm install
+# Install additional OS packages.
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends jq bash
 
-WORKDIR /app
+# We do this to get a yq binary from the published container, for the correct architecture we're building here
+COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
+
+RUN mkdir -p /scripts
+COPY ./apply-webapp-config.sh /scripts
+COPY ./start-serving-app.sh /scripts
+
+# Configure the local npm registry
+RUN npm config set @cerc-io:registry ${CERC_NPM_REGISTRY_URL}
+
+RUN mkdir -p /config
+
+# Install simple web server for now (use nginx perhaps later)
+RUN yarn global add http-server
+
+# Globally install the payload web app package
+RUN yarn global add @cerc-io/mobymask-ui
+
+# Expose port for http
+EXPOSE 80
+
+CMD ["/scripts/start-serving-app.sh"]

app/data/container-build/cerc-mobymask-ui/apply-webapp-config.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+if [[ $# -ne 3 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+config_file_name=$1
+webapp_files_dir=$2
+config_prefix=$3
+if ! [[ -f ${config_file_name} ]]; then
+    echo "Config file ${config_file_name} does not exist" >&2
+    exit 1
+fi
+if ! [[ -d ${webapp_files_dir} ]]; then
+    echo "Webapp directory ${webapp_files_dir} does not exist" >&2
+    exit 1
+fi
+# First some magic using sed to translate our yaml config file into an array of key value pairs like:
+# ${config_prefix}<path-through-objects>=<value>
+# sed "s/'//g" is used to remove single quote for relayNodes value
+readarray -t config_kv_pair_array < <( sed -E 's/([^:]+):\s*(.*)/\1=\2/g' ${config_file_name} | sed "s/'//g" |  sed "s/^/${config_prefix}_/" )
+declare -p config_kv_pair_array
+# Then iterate over that kv array making the template substitution in our web app files
+for kv_pair_string in "${config_kv_pair_array[@]}"
+do
+    kv_pair=(${kv_pair_string//=/ })
+    template_string_to_replace=${kv_pair[0]}
+    template_value_to_substitute=${kv_pair[1]}
+    # Run find and sed to do the substitution of one variable over all files
+    # See: https://stackoverflow.com/a/21479607/1701505
+    echo "Substituting: ${template_string_to_replace} = ${template_value_to_substitute}"
+
+    # TODO: Pass keys to be replaced without double quotes
+    if [[ "$template_string_to_replace" =~ ^${config_prefix}_(relayNodes|chainId)$ ]]; then
+        find ${webapp_files_dir} -type f -exec sed -i 's#"'"${template_string_to_replace}"'"#'"${template_value_to_substitute}"'#g' {} +
+    else
+        # Note: we do not escape our strings, on the expectation they do not container the '#' char.
+        find ${webapp_files_dir} -type f -exec sed -i 's#'${template_string_to_replace}'#'${template_value_to_substitute}'#g' {} +
+    fi
+done

app/data/container-build/cerc-mobymask-ui/build.sh

@@ -6,4 +6,7 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/mobymask-ui:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mobymask-ui
+CERC_NPM_REGISTRY_URL="https://git.vdb.to/api/packages/cerc-io/npm/"
+
+docker build -t cerc/mobymask-ui:local  ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile \
+  --build-arg CERC_NPM_REGISTRY_URL ${SCRIPT_DIR}

app/data/container-build/cerc-mobymask-ui/start-serving-app.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+# TODO: Don't hard wire this:
+webapp_files_dir=/usr/local/share/.config/yarn/global/node_modules/@cerc-io/mobymask-ui/build
+/scripts/apply-webapp-config.sh /config/config.yml ${webapp_files_dir} MOBYMASK_HOSTED_CONFIG
+http-server -p 80 ${webapp_files_dir}

app/data/container-build/cerc-react-peer/Dockerfile

@@ -1,14 +1,30 @@
 FROM node:18.15.0-alpine3.16
 
-RUN apk --update --no-cache add make git python3 jq
+# This container pulls npm package from a registry configured via env var
+ARG CERC_NPM_REGISTRY_URL
 
-WORKDIR /app
+RUN apk --update --no-cache add make git python3 jq bash
 
-COPY . .
+# We do this to get a yq binary from the published container, for the correct architecture we're building here
+COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
 
-RUN yarn global add serve
+RUN mkdir -p /scripts
+COPY ./apply-webapp-config.sh /scripts
+COPY ./start-serving-app.sh /scripts
 
-RUN echo "Building react-peer" && \
-    yarn install --ignore-scripts && yarn build --ignore @cerc-io/test-app
+# Configure the local npm registry
+RUN npm config set @cerc-io:registry ${CERC_NPM_REGISTRY_URL}
 
-WORKDIR /app
+RUN mkdir -p /config
+
+# Install simple web server for now (use nginx perhaps later)
+RUN yarn global add http-server
+
+# Globally install the payload web app package
+RUN yarn global add @cerc-io/test-app
+
+
+# Expose port for http
+EXPOSE 80
+
+CMD ["/scripts/start-serving-app.sh"]

app/data/container-build/cerc-react-peer/apply-webapp-config.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+if [[ $# -ne 3 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+config_file_name=$1
+webapp_files_dir=$2
+config_prefix=$3
+if ! [[ -f ${config_file_name} ]]; then
+    echo "Config file ${config_file_name} does not exist" >&2
+    exit 1
+fi
+if ! [[ -d ${webapp_files_dir} ]]; then
+    echo "Webapp directory ${webapp_files_dir} does not exist" >&2
+    exit 1
+fi
+# First some magic using sed to translate our yaml config file into an array of key value pairs like:
+# ${config_prefix}<path-through-objects>=<value>
+# sed "s/'//g" is used to remove single quote for relayNodes value
+readarray -t config_kv_pair_array < <( sed -E 's/([^:]+):\s*(.*)/\1=\2/g' ${config_file_name} | sed "s/'//g" |  sed "s/^/${config_prefix}_/" )
+declare -p config_kv_pair_array
+# Then iterate over that kv array making the template substitution in our web app files
+for kv_pair_string in "${config_kv_pair_array[@]}"
+do
+    kv_pair=(${kv_pair_string//=/ })
+    template_string_to_replace=${kv_pair[0]}
+    template_value_to_substitute=${kv_pair[1]}
+    # Run find and sed to do the substitution of one variable over all files
+    # See: https://stackoverflow.com/a/21479607/1701505
+    echo "Substituting: ${template_string_to_replace} = ${template_value_to_substitute}"
+
+    # TODO: Pass keys to be replaced without double quotes
+    if [[ "$template_string_to_replace" == "${config_prefix}_relayNodes" ]]; then
+        find ${webapp_files_dir} -type f -exec sed -i 's#"'"${template_string_to_replace}"'"#'"${template_value_to_substitute}"'#g' {} +
+    else
+        # Note: we do not escape our strings, on the expectation they do not container the '#' char.
+        find ${webapp_files_dir} -type f -exec sed -i 's#'${template_string_to_replace}'#'${template_value_to_substitute}'#g' {} +
+    fi
+done

app/data/container-build/cerc-react-peer/build.sh

@@ -6,4 +6,7 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/react-peer:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/react-peer
+CERC_NPM_REGISTRY_URL="https://git.vdb.to/api/packages/cerc-io/npm/"
+
+docker build -t cerc/react-peer:local  ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile \
+  --build-arg CERC_NPM_REGISTRY_URL ${SCRIPT_DIR}

app/data/container-build/cerc-react-peer/start-serving-app.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+# TODO: Don't hard wire this:
+webapp_files_dir=/usr/local/share/.config/yarn/global/node_modules/@cerc-io/test-app/build
+/scripts/apply-webapp-config.sh /config/config.yml ${webapp_files_dir} MOBYMASK_HOSTED_CONFIG
+http-server -p 80 ${webapp_files_dir}

app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile

@@ -17,4 +17,4 @@ COPY . .
 RUN echo "Building watcher-ts" && \
     yarn && yarn build
 
-WORKDIR /app/packages/mobymask-v2-watcher
+WORKDIR /app

app/data/container-build/cerc-watcher-mobymask-v2/build.sh

@@ -6,4 +6,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/watcher-mobymask-v2:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/watcher-ts
+docker build -t cerc/watcher-mobymask-v2:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mobymask-v2-watcher-ts

app/data/container-build/cerc-watcher-ts/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:16.17.1-alpine3.16
+
+RUN apk --update --no-cache add git python3 alpine-sdk jq
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building watcher-ts" && \
+    yarn && yarn build

app/data/container-build/cerc-watcher-ts/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-erc20
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-ts:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/watcher-ts

app/data/container-image-list.txt

@@ -14,6 +14,7 @@ cerc/laconic-registry-cli
 cerc/laconic-console-host
 cerc/fixturenet-eth-geth
 cerc/fixturenet-eth-lighthouse
+cerc/watcher-ts
 cerc/watcher-mobymask
 cerc/watcher-erc20
 cerc/watcher-erc721

app/data/pod-list.txt

@@ -10,6 +10,7 @@ laconicd
 fixturenet-laconicd
 fixturenet-eth
 fixturenet-eth-metrics
+watcher-ts
 watcher-mobymask
 watcher-erc20
 watcher-erc721

app/data/repository-list.txt

@@ -12,6 +12,7 @@ cerc-io/laconic-registry-cli
 cerc-io/laconic-console
 cerc-io/mobymask-watcher
 cerc-io/watcher-ts
+cerc-io/mobymask-v2-watcher-ts
 cerc-io/react-peer
 cerc-io/mobymask-ui
 cerc-io/MobyMask

app/data/stacks/fixturenet-eth/README.md

@@ -3,15 +3,18 @@
 Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#user-mode)):
 
 ## Clone required repositories
+
 ```
 $ laconic-so --stack fixturenet-eth setup-repositories
 ```
 
 ## Build the fixturenet-eth containers
+
 ```
 $ laconic-so --stack fixturenet-eth build-containers
 ```
-This should create several container images in the local image registry: 
+
+This should create several container images in the local image registry:
 
 * cerc/go-ethereum
 * cerc/lighthouse
@@ -19,6 +22,7 @@ This should create several container images in the local image registry:
 * cerc/fixturenet-eth-lighthouse
 
 ## Deploy the stack
+
 ```
 $ laconic-so --stack fixturenet-eth deploy up
 ```
@@ -57,7 +61,7 @@ Several other containers can used with the basic `fixturenet-eth`:
 * `eth-probe` (captures eth1 tx gossip)
 * `keycloak` (nginx proxy with keycloak auth for API authentication)
 * `tx-spammer` (generates and sends automated transactions to the fixturenet)
-	
+
 It is not necessary to use them all at once, but a complete example follows:
 
 ```
@@ -99,4 +103,22 @@ keycloak-db-1                          0.0.0.0:55850->5432/tcp
 keycloak-nginx-1                       0.0.0.0:55859->80/tcp
 migrations-1
 tx-spammer-1
-```
+```
+
+## Clean up
+
+Stop all services running in the background:
+
+```bash
+$ laconic-so --stack fixturenet-eth deploy down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+$ docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_bootnode_lighthouse_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data|.*fixturenet_geth_accounts"
+
+# Remove all the listed volumes
+$ docker volume rm $(docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_bootnode_lighthouse_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data|.*fixturenet_geth_accounts")
+```

app/data/stacks/fixturenet-laconic-loaded/README.md

@@ -14,7 +14,7 @@ $ chmod +x ./laconic-so
 $ export PATH=$PATH:$(pwd)  # Or move laconic-so to ~/bin or your favorite on-path directory
 ```
 ## 2. Prepare the local build environment
-Note that this step needs only to be done once on a new machine. 
+Note that this step needs only to be done once on a new machine.
 Detailed instructions can be found [here](../build-support/README.md). For the impatient run these commands:
 ```
 $ laconic-so --stack build-support build-containers --exclude cerc/builder-gerbil

app/data/stacks/fixturenet-optimism/README.md

@@ -91,6 +91,9 @@ docker volume rm $(docker volume ls -q --filter "name=.*fixturenet_geth_accounts
   ```
 
 * This means that the data directory that `op-geth` is using is corrupted and needs to be reinitialized; the containers `op-geth`, `op-node` and `op-batcher` need to be started afresh:
+
+  WARNING: This will reset the L2 chain; consequently, all the data on it will be lost
+
   * Stop and remove the concerned containers:
 
     ```bash

app/data/stacks/mobymask-v2/README.md

@@ -23,15 +23,7 @@ Checkout to the required versions and branches in repos
 ```bash
 # watcher-ts
 cd ~/cerc/watcher-ts
-git checkout v0.2.35
-
-# react-peer
-cd ~/cerc/react-peer
-git checkout v0.2.31
-
-# mobymask-ui
-cd ~/cerc/mobymask-ui
-git checkout laconic
+git checkout v0.2.39
 
 # MobyMask
 cd ~/cerc/MobyMask
@@ -73,13 +65,13 @@ Deploy the stack:
 Find the watcher container's id and export it for later use:
 
 ```bash
-export CONTAINER_ID=$(docker ps -q --filter "name=mobymask-watcher-server")
+export CONTAINER_ID=$(docker ps -q --filter "name=peer-tests")
 ```
 
 Run the peer tests:
 
 ```bash
-docker exec -w /app/packages/peer $CONTAINER_ID yarn test
+docker exec $CONTAINER_ID yarn test
 ```
 
 ## Web Apps

app/data/stacks/mobymask-v2/demo.md

@@ -66,7 +66,7 @@
   * Get the deployed contract address:
 
     ```bash
-    docker exec -it $(docker ps -aq --filter name="mobymask-app") cat src/config.json
+    docker exec -it $(docker ps -aq --filter name="mobymask-app") cat /config/config.yml
     ```
 
     The value of `address` field is the deployed contract address

app/data/stacks/mobymask-v2/stack.yml

@@ -6,8 +6,7 @@ repos:
   - ethereum-optimism/optimism
   - ethereum-optimism/op-geth
   - cerc-io/watcher-ts
-  - cerc-io/react-peer
-  - cerc-io/mobymask-ui
+  - cerc-io/mobymask-v2-watcher-ts
   - cerc-io/MobyMask
 containers:
   - cerc/go-ethereum
@@ -19,6 +18,7 @@ containers:
   - cerc/optimism-l2geth
   - cerc/optimism-op-batcher
   - cerc/optimism-op-node
+  - cerc/watcher-ts
   - cerc/watcher-mobymask-v2
   - cerc/react-peer
   - cerc/mobymask-ui

app/data/stacks/mobymask-v2/web-apps.md

@@ -6,30 +6,10 @@ Instructions to setup and deploy MobyMask and Peer Test web apps
 
 Prerequisite: Watcher with GQL and relay node endpoints
 
-Clone required repositories:
-
-```bash
-laconic-so --stack mobymask-v2 setup-repositories --include cerc-io/react-peer,cerc-io/mobymask-ui
-
-# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
-```
-
-Checkout to the required versions and branches in repos:
-
-```bash
-# react-peer
-cd ~/cerc/react-peer
-git checkout v0.2.31
-
-# mobymask-ui
-cd ~/cerc/mobymask-ui
-git checkout laconic
-```
-
 Build the container images:
 
 ```bash
-laconic-so --stack mobymask-v2 build-containers --include cerc/react-peer-v2,cerc/mobymask-ui
+laconic-so --stack mobymask-v2 build-containers --include cerc/react-peer,cerc/mobymask-ui
 ```
 
 This should create the required docker images in the local image registry

app/setup_repositories.py

@@ -175,4 +175,8 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches_file):
                         git_repo.git.checkout(branch_to_checkout)
 
     for repo in repos:
-        process_repo(repo)
+        try:
+            process_repo(repo)
+        except git.exc.GitCommandError as error:
+            print(f"\n******* git command returned error exit status:\n{error}")
+            sys.exit(1)

scripts/build_shiv_package.sh

@@ -1,4 +1,4 @@
 # Builds the shiv "package" for distribution
 mkdir -p ./package
-version_string=$( ./app/data/version.txt )
+version_string=$( ./scripts/create_build_tag_file.sh )
 shiv -c laconic-so -o package/laconic-so-${version_string} .

scripts/developer-mode-setup.sh

@@ -7,5 +7,6 @@ if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
 fi
 python3 -m venv venv
 source ./venv/bin/activate
-pip install --editable .
+python3 -m pip install --upgrade pip setuptools wheel
 pip install shiv
+pip install --editable .