Add ability to run Optimism fixturenet with external L1 endpoint (#273)

* Remove unnecessary todos

* Set option to log commands in shell scripts

* Replace fixturenet-eth dependency with wait on endpoint

* Skip lighthouse node dependency check

* Update all services in the stack

* Use debug flag to enable shell commands logging

* Add bash in op-batcher container

* Update mobymask-v2 instructions

* Update fixturenet-optimism instructions

* Add descriptions for services

* Move ts files to container-build

* Take L1 RPC endpoint from the env file

* Add dev mode restriction for editing env file

Former-commit-id: 2515878eeb3456147d4e50e1b39c98f618d1a3d3

app/data/config/fixturenet-optimism/generate-l2-config.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
 
 op-node genesis l2 \
   --deploy-config /contracts-bedrock/deploy-config/getting-started.json \

app/data/config/fixturenet-optimism/l1-params.env

@@ -0,0 +1,9 @@
+# Change if pointing to an external L1 endpoint
+L1_RPC="http://fixturenet-eth-geth-1:8545"
+L1_CHAIN_ID=1212
+L1_HOST="fixturenet-eth-geth-1"
+L1_PORT=8545
+L1_ADDRESS=
+L1_PRIV_KEY=
+L1_ADDRESS_2=
+L1_PRIV_KEY_2=

app/data/config/fixturenet-optimism/optimism-contracts/rekey-json.ts

@@ -1,28 +0,0 @@
-import fs from 'fs'
-
-import { task } from 'hardhat/config'
-import { hdkey } from 'ethereumjs-wallet'
-import * as bip39 from 'bip39'
-
-task('rekey-json', 'Generates a new set of keys for a test network')
-  .addParam('output', 'JSON file to output accounts to')
-  .setAction(async ({ output: outputFile }) => {
-    const mnemonic = bip39.generateMnemonic()
-    const pathPrefix = "m/44'/60'/0'/0"
-    const labels = ['Admin', 'Proposer', 'Batcher', 'Sequencer']
-    const hdwallet = hdkey.fromMasterSeed(await bip39.mnemonicToSeed(mnemonic))
-
-    const output = {}
-
-    for (let i = 0; i < labels.length; i++) {
-      const label = labels[i]
-      const wallet = hdwallet.derivePath(`${pathPrefix}/${i}`).getWallet()
-      const addr = '0x' + wallet.getAddress().toString('hex')
-      const pk = wallet.getPrivateKey().toString('hex')
-
-      output[label] = { address: addr, privateKey: pk }
-    }
-
-    fs.writeFileSync(outputFile, JSON.stringify(output, null, 2))
-    console.log(`L2 account keys written to ${outputFile}`)
-  })

app/data/config/fixturenet-optimism/optimism-contracts/run.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
 
 # TODO Support restarts; fixturenet-eth-geth currently starts fresh on a restart
 # Exit if a deployment already exists (on restarts)
@@ -8,12 +11,14 @@ set -e
 #     exit 0
 # fi
 
+echo "Using L1 RPC endpoint ${L1_RPC}"
+
 # Append tasks/index.ts file
 echo "import './rekey-json'" >> tasks/index.ts
 echo "import './send-balance'" >> tasks/index.ts
 
 # Update the chainId in the hardhat config
-sed -i "/getting-started/ {n; s/.*chainId.*/      chainId: $CHAIN_ID,/}" hardhat.config.ts
+sed -i "/getting-started/ {n; s/.*chainId.*/      chainId: $L1_CHAIN_ID,/}" hardhat.config.ts
 
 # Generate the L2 account addresses
 yarn hardhat rekey-json --output /l2-accounts/keys.json
@@ -29,11 +34,27 @@ BATCHER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Batcher.address')
 SEQUENCER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Sequencer.address')
 
 # Read the private key of L1 accounts
-# TODO: Take from env if /geth-accounts volume doesn't exist to allow using separately running L1
-L1_ADDRESS=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 2)
-L1_PRIV_KEY=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 3)
-L1_ADDRESS_2=$(awk -F, 'NR==2{print $(NF-1)}' /geth-accounts/accounts.csv)
-L1_PRIV_KEY_2=$(awk -F, 'NR==2{print $NF}' /geth-accounts/accounts.csv)
+if [ -f /geth-accounts/accounts.csv ]; then
+  echo "Using L1 account credentials from the mounted volume"
+  L1_ADDRESS=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 2)
+  L1_PRIV_KEY=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 3)
+  L1_ADDRESS_2=$(awk -F, 'NR==2{print $(NF-1)}' /geth-accounts/accounts.csv)
+  L1_PRIV_KEY_2=$(awk -F, 'NR==2{print $NF}' /geth-accounts/accounts.csv)
+else
+  echo "Using L1 account credentials from env"
+fi
+
+# Select a finalized L1 block as the starting point for roll ups
+until FINALIZED_BLOCK=$(cast block finalized --rpc-url "$L1_RPC"); do
+    echo "Waiting for a finalized L1 block to exist, retrying after 10s"
+    sleep 10
+done
+
+L1_BLOCKNUMBER=$(echo "$FINALIZED_BLOCK" | awk '/number/{print $2}')
+L1_BLOCKHASH=$(echo "$FINALIZED_BLOCK" | awk '/hash/{print $2}')
+L1_BLOCKTIMESTAMP=$(echo "$FINALIZED_BLOCK" | awk '/timestamp/{print $2}')
+
+echo "Selected L1 block ${L1_BLOCKNUMBER} as the starting block for roll ups"
 
 # Send balances to the above L2 addresses
 yarn hardhat send-balance --to "${ADMIN_ADDRESS}" --amount 2 --private-key "${L1_PRIV_KEY}" --network getting-started
@@ -42,19 +63,9 @@ yarn hardhat send-balance --to "${BATCHER_ADDRESS}" --amount 1000 --private-key
 
 echo "Balances sent to L2 accounts"
 
-# Select a finalized L1 block as the starting point for roll ups
-# TODO Use web3.js to get the latest finalized block
-until CAST_OUTPUT=$(cast block finalized --rpc-url "$L1_RPC"); do
-    echo "Waiting for a finalized L1 block to exist, retrying after 10s"
-    sleep 10
-done
-
-L1_BLOCKHASH=$(echo "$CAST_OUTPUT" | awk '/hash/{print $2}')
-L1_BLOCKTIMESTAMP=$(echo "$CAST_OUTPUT" | awk '/timestamp/{print $2}')
-
 # Update the deployment config
 sed -i 's/"l2OutputOracleStartingTimestamp": TIMESTAMP/"l2OutputOracleStartingTimestamp": '"$L1_BLOCKTIMESTAMP"'/g' deploy-config/getting-started.json
-jq --arg chainid "$CHAIN_ID" '.l1ChainID = ($chainid | tonumber)' deploy-config/getting-started.json > tmp.json && mv tmp.json deploy-config/getting-started.json
+jq --arg chainid "$L1_CHAIN_ID" '.l1ChainID = ($chainid | tonumber)' deploy-config/getting-started.json > tmp.json && mv tmp.json deploy-config/getting-started.json
 
 node update-config.js deploy-config/getting-started.json "$ADMIN_ADDRESS" "$PROPOSER_ADDRESS" "$BATCHER_ADDRESS" "$SEQUENCER_ADDRESS" "$L1_BLOCKHASH"
 

app/data/config/fixturenet-optimism/optimism-contracts/send-balance.ts

@@ -1,22 +0,0 @@
-import { task } from 'hardhat/config'
-import '@nomiclabs/hardhat-ethers'
-import { ethers } from 'ethers'
-
-task('send-balance', 'Sends Ether to a specified Ethereum account')
-  .addParam('to', 'The Ethereum address to send Ether to')
-  .addParam('amount', 'The amount of Ether to send, in Ether')
-  .addParam('privateKey', 'The private key of the sender')
-  .setAction(async ({ to, amount, privateKey }, {}) => {
-    // Open the wallet using sender's private key
-    const provider = new ethers.providers.JsonRpcProvider(`${process.env.L1_RPC}`)
-    const wallet = new ethers.Wallet(privateKey, provider)
-
-    // Send amount to the specified address
-    const tx = await wallet.sendTransaction({
-      to,
-      value: ethers.utils.parseEther(amount),
-    })
-
-    console.log(`Balance sent to: ${to}, from: ${wallet.address}`)
-    console.log(`Transaction hash: ${tx.hash}`)
-  })

app/data/config/fixturenet-optimism/run-op-batcher.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
 
 # Get BACTHER_KEY from keys.json
 BATCHER_KEY=$(jq -r '.Batcher.privateKey' /l2-accounts/keys.json | tr -d '"')

app/data/config/fixturenet-optimism/run-op-geth.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
 
 mkdir datadir
 

app/data/config/fixturenet-optimism/run-op-node.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
 
 # Get SEQUENCER KEY from keys.json
 SEQUENCER_KEY=$(jq -r '.Sequencer.privateKey' /l2-accounts/keys.json | tr -d '"')

app/data/config/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi