[WIP] Handle restarts in fixturenet-eth stack (#324)

* Use mounted volumes for data in geth nodes

* Use mounted volumes for data in lighthouse nodes

* Avoid resetting genesis time in a lighthouse node on restart

* Mount parent datadir for lighthouse nodes

* Trap signals on shutdown and clean up in lighthouse nodes

* Allow stalled sync in lighthouse beacon nodes

* Gracefully shutdown geth nodes

* Add clean up instructions

* Gracefully shutdown lighthouse boot node

app/data/container-build/cerc-fixturenet-eth-geth/Dockerfile

@@ -22,6 +22,6 @@ COPY run-el.sh /opt/testnet/run.sh
 RUN cd /opt/testnet && make genesis-el
 
 COPY --from=geth  /usr/local/bin/geth /usr/local/bin/
-RUN geth init /opt/testnet/build/el/geth.json && rm -f ~/.ethereum/geth/nodekey
+RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
 
 ENTRYPOINT ["/opt/testnet/run.sh"]

app/data/container-build/cerc-fixturenet-eth-geth/genesis/accounts/import_keys.sh

@@ -12,6 +12,6 @@ for line in `cat ../build/el/accounts.csv`; do
 
   echo ""
   echo "$ADDRESS"
-  geth account import --password .pw.$$ .key.$$
+  geth account import --datadir=~/ethdata --password .pw.$$ .key.$$
   rm -f .pw.$$ .key.$$
 done

app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -18,17 +18,35 @@ if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
     START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --"
 fi
 
+# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script
+cleanup() {
+    echo "Signal received, cleaning up..."
+
+    # Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process)
+    pkill -P ${geth_pid}
+    sleep 2
+    kill $(jobs -p)
+
+    wait
+    echo "Done"
+}
+trap 'cleanup' SIGINT SIGTERM
+
 if [ "true" == "$RUN_BOOTNODE" ]; then
     $START_CMD \
+       --datadir=~/ethdata \
       --nodekeyhex="${BOOTNODE_KEY}" \
       --nodiscover \
       --ipcdisable \
       --networkid=${NETWORK_ID} \
-      --netrestrict="${NETRESTRICT}"
+      --netrestrict="${NETRESTRICT}" \
+      &
+
+    geth_pid=$!
 else
     cd /opt/testnet/accounts
     ./import_keys.sh
-    
+
     echo -n "$JWT" > /opt/testnet/build/el/jwtsecret
 
     if [ "$CERC_RUN_STATEDIFF" == "detect" ] && [ -n "$CERC_STATEDIFF_DB_HOST" ]; then
@@ -74,6 +92,7 @@ else
     fi
 
     $START_CMD \
+      --datadir=~/ethdata \
       --bootnodes="${ENODE}" \
       --allow-insecure-unlock \
       --http \
@@ -101,5 +120,10 @@ else
       --metrics.addr="0.0.0.0" \
       --verbosity=${CERC_GETH_VERBOSITY:-3} \
       --vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
-      --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS}
+      --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS} \
+      &
+
+    geth_pid=$!
 fi
+
+wait $geth_pid