Fetch geth accounts using an exposed endpoint (#357)

* Fetch account creds served by geth service * Use fetched account creds in mobymask-v2 stack Former-commit-id: eb777b0b47
2023-04-20 15:12:59 +05:30 · 2023-04-20 15:12:59 +05:30 · 363a0b733f
commit 363a0b733f
parent 3d75523d73
16 changed files with 65 additions and 48 deletions
--- a/app/data/compose/docker-compose-fixturenet-eth.yml
+++ b/app/data/compose/docker-compose-fixturenet-eth.yml
@ -27,7 +27,6 @@ services:
      - ../config/fixturenet-eth/fixturenet-eth.env
    image: cerc/fixturenet-eth-geth:local
    volumes:
-      - fixturenet_geth_accounts:/opt/testnet/build/el
      - fixturenet_eth_geth_1_data:/root/ethdata
    healthcheck:
      test: ["CMD", "nc", "-v", "localhost", "8545"]
@ -116,7 +115,6 @@ services:
        condition: service_healthy

 volumes:
-  fixturenet_geth_accounts:
  fixturenet_eth_bootnode_geth_data:
  fixturenet_eth_geth_1_data:
  fixturenet_eth_geth_2_data:
--- a/app/data/compose/docker-compose-fixturenet-optimism.yml
+++ b/app/data/compose/docker-compose-fixturenet-optimism.yml
@ -13,6 +13,7 @@ services:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
      CERC_L1_CHAIN_ID: ${CERC_L1_CHAIN_ID}
      CERC_L1_RPC: ${CERC_L1_RPC}
+      CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL}
      CERC_L1_ADDRESS: ${CERC_L1_ADDRESS}
      CERC_L1_PRIV_KEY: ${CERC_L1_PRIV_KEY}
      CERC_L1_ADDRESS_2: ${CERC_L1_ADDRESS_2}
@ -27,7 +28,6 @@ services:
      - ../container-build/cerc-optimism-contracts/hardhat-tasks/send-balance.ts:/app/packages/contracts-bedrock/tasks/send-balance.ts
      - ../config/fixturenet-optimism/optimism-contracts/update-config.js:/app/packages/contracts-bedrock/update-config.js
      - ../config/fixturenet-optimism/optimism-contracts/run.sh:/app/packages/contracts-bedrock/run.sh
-      - fixturenet_geth_accounts:/geth-accounts:ro
      - l2_accounts:/l2-accounts
      - l1_deployment:/app/packages/contracts-bedrock
    extra_hosts:
@ -126,7 +126,6 @@ services:
    - "host.docker.internal:host-gateway"

 volumes:
-  fixturenet_geth_accounts:
  l1_deployment:
  l2_accounts:
  l2_config:
--- a/app/data/compose/docker-compose-watcher-mobymask-v2.yml
+++ b/app/data/compose/docker-compose-watcher-mobymask-v2.yml
@ -34,6 +34,7 @@ services:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
      ENV: "PROD"
      CERC_L2_GETH_RPC: ${CERC_L2_GETH_RPC}
+      CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL}
      CERC_PRIVATE_KEY_DEPLOYER: ${CERC_PRIVATE_KEY_DEPLOYER}
      CERC_MOBYMASK_APP_BASE_URI: ${CERC_MOBYMASK_APP_BASE_URI}
      CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
@ -50,7 +51,6 @@ services:
      - ../config/watcher-mobymask-v2/secrets-template.json:/app/packages/server/secrets-template.json
      - ../config/watcher-mobymask-v2/deploy-and-generate-invite.sh:/app/packages/server/deploy-and-generate-invite.sh
      - mobymask_deployment:/app/packages/server
-      - fixturenet_geth_accounts:/geth-accounts:ro
    extra_hosts:
      - "host.docker.internal:host-gateway"

@ -83,6 +83,7 @@ services:
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
      CERC_L2_GETH_RPC: ${CERC_L2_GETH_RPC}
+      CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL}
      CERC_PRIVATE_KEY_PEER: ${CERC_PRIVATE_KEY_PEER}
      CERC_RELAY_PEERS: ${CERC_RELAY_PEERS}
      CERC_RELAY_ANNOUNCE_DOMAIN: ${CERC_RELAY_ANNOUNCE_DOMAIN}
@ -94,7 +95,6 @@ services:
      - ../config/watcher-mobymask-v2/start-server.sh:/app/start-server.sh
      - peers_ids:/app/peers
      - mobymask_deployment:/server
-      - fixturenet_geth_accounts:/geth-accounts:ro
    # Expose GQL, metrics and relay node ports
    ports:
      - "0.0.0.0:3001:3001"
@ -135,4 +135,3 @@ volumes:
  mobymask_watcher_db_data:
  peers_ids:
  mobymask_deployment:
-  fixturenet_geth_accounts:
--- a/app/data/config/fixturenet-optimism/l1-params.env
+++ b/app/data/config/fixturenet-optimism/l1-params.env
@ -6,9 +6,7 @@ DEFAULT_CERC_L1_RPC="http://fixturenet-eth-geth-1:8545"
 DEFAULT_CERC_L1_HOST="fixturenet-eth-geth-1"
 DEFAULT_CERC_L1_PORT=8545

-# Credentials for accounts on L1 to send balance to Optimism Proxy contract from
+# URL to get CSV with credentials for accounts on L1
+# that are used to send balance to Optimism Proxy contract
 # (enables them to do transactions on L2)
-DEFAULT_CERC_L1_ADDRESS=
-DEFAULT_CERC_L1_PRIV_KEY=
-DEFAULT_CERC_L1_ADDRESS_2=
-DEFAULT_CERC_L1_PRIV_KEY_2=
+DEFAULT_CERC_L1_ACCOUNTS_CSV_URL="http://fixturenet-eth-bootnode-geth:9898/accounts.csv"
--- a/app/data/config/fixturenet-optimism/optimism-contracts/run.sh
+++ b/app/data/config/fixturenet-optimism/optimism-contracts/run.sh
@ -7,10 +7,7 @@ fi
 CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
 CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"

-CERC_L1_ADDRESS="${CERC_L1_ADDRESS:-${DEFAULT_CERC_L1_ADDRESS}}"
-CERC_L1_PRIV_KEY="${CERC_L1_PRIV_KEY:-${DEFAULT_CERC_L1_PRIV_KEY}}"
-CERC_L1_ADDRESS_2="${CERC_L1_ADDRESS_2:-${DEFAULT_CERC_L1_ADDRESS_2}}"
-CERC_L1_PRIV_KEY_2="${CERC_L1_PRIV_KEY_2:-${DEFAULT_CERC_L1_PRIV_KEY_2}}"
+CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"

 echo "Using L1 RPC endpoint ${CERC_L1_RPC}"

@ -62,15 +59,21 @@ PROPOSER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Proposer.address')
 BATCHER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Batcher.address')
 SEQUENCER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Sequencer.address')

-# Read the private key of L1 accounts
-if [ -f /geth-accounts/accounts.csv ]; then
-  echo "Using L1 account credentials from the mounted volume"
+# Get the private keys of L1 accounts
+if [ -n "$CERC_L1_ACCOUNTS_CSV_URL" ] && \
+  l1_accounts_response=$(curl -L --write-out '%{http_code}' --silent --output /dev/null "$CERC_L1_ACCOUNTS_CSV_URL") && \
+  [ "$l1_accounts_response" -eq 200 ];
+then
+  echo "Fetching L1 account credentials using provided URL"
+  mkdir -p /geth-accounts
+  wget -O /geth-accounts/accounts.csv "$CERC_L1_ACCOUNTS_CSV_URL"
+
  CERC_L1_ADDRESS=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 2)
  CERC_L1_PRIV_KEY=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 3)
  CERC_L1_ADDRESS_2=$(awk -F, 'NR==2{print $(NF-1)}' /geth-accounts/accounts.csv)
  CERC_L1_PRIV_KEY_2=$(awk -F, 'NR==2{print $NF}' /geth-accounts/accounts.csv)
 else
-  echo "Using L1 account credentials from env"
+  echo "Couldn't fetch L1 account credentials, using them from env"
 fi

 # Select a finalized L1 block as the starting point for roll ups
--- a/app/data/config/watcher-mobymask-v2/deploy-and-generate-invite.sh
+++ b/app/data/config/watcher-mobymask-v2/deploy-and-generate-invite.sh
@ -5,19 +5,25 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi

 CERC_L2_GETH_RPC="${CERC_L2_GETH_RPC:-${DEFAULT_CERC_L2_GETH_RPC}}"
-CERC_PRIVATE_KEY_DEPLOYER="${CERC_PRIVATE_KEY_DEPLOYER:-${DEFAULT_CERC_PRIVATE_KEY_DEPLOYER}}"
+CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"

 CERC_MOBYMASK_APP_BASE_URI="${CERC_MOBYMASK_APP_BASE_URI:-${DEFAULT_CERC_MOBYMASK_APP_BASE_URI}}"
 CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRACT}}"

 echo "Using L2 RPC endpoint ${CERC_L2_GETH_RPC}"

-if [ -f /geth-accounts/accounts.csv ]; then
-  echo "Using L1 private key from the mounted volume"
-  # Read the private key of L1 account to deploy contract
+if [ -n "$CERC_L1_ACCOUNTS_CSV_URL" ] && \
+  l1_accounts_response=$(curl -L --write-out '%{http_code}' --silent --output /dev/null "$CERC_L1_ACCOUNTS_CSV_URL") && \
+  [ "$l1_accounts_response" -eq 200 ];
+then
+  echo "Fetching L1 account credentials using provided URL"
+  mkdir -p /geth-accounts
+  wget -O /geth-accounts/accounts.csv "$CERC_L1_ACCOUNTS_CSV_URL"
+
+  # Read the private key of an L1 account to deploy contract
  CERC_PRIVATE_KEY_DEPLOYER=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 3)
 else
-  echo "Using CERC_PRIVATE_KEY_DEPLOYER from env"
+  echo "Couldn't fetch L1 account credentials, using CERC_PRIVATE_KEY_DEPLOYER from env"
 fi

 # Set the private key
--- a/app/data/config/watcher-mobymask-v2/optimism-params.env
+++ b/app/data/config/watcher-mobymask-v2/optimism-params.env
@ -8,6 +8,5 @@ DEFAULT_CERC_L2_GETH_PORT=8545
 DEFAULT_CERC_L2_NODE_HOST="op-node"
 DEFAULT_CERC_L2_NODE_PORT=8547

-# Credentials for accounts to perform txs on L2
-DEFAULT_CERC_PRIVATE_KEY_DEPLOYER=
-DEFAULT_CERC_PRIVATE_KEY_PEER=
+# URL to get CSV with credentials for accounts on L1 to perform txs on L2
+DEFAULT_CERC_L1_ACCOUNTS_CSV_URL="http://fixturenet-eth-bootnode-geth:9898/accounts.csv"
--- a/app/data/config/watcher-mobymask-v2/start-server.sh
+++ b/app/data/config/watcher-mobymask-v2/start-server.sh
@ -5,7 +5,7 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi

 CERC_L2_GETH_RPC="${CERC_L2_GETH_RPC:-${DEFAULT_CERC_L2_GETH_RPC}}"
-CERC_PRIVATE_KEY_PEER="${CERC_PRIVATE_KEY_PEER:-${DEFAULT_CERC_PRIVATE_KEY_PEER}}"
+CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"

 CERC_RELAY_PEERS="${CERC_RELAY_PEERS:-${DEFAULT_CERC_RELAY_PEERS}}"
 CERC_RELAY_ANNOUNCE_DOMAIN="${CERC_RELAY_ANNOUNCE_DOMAIN:-${DEFAULT_CERC_RELAY_ANNOUNCE_DOMAIN}}"
@ -24,12 +24,18 @@ else
  CONTRACT_ADDRESS=$(jq -r '.address' /server/config.json | tr -d '"')
 fi

-if [ -f /geth-accounts/accounts.csv ]; then
-  echo "Using L1 private key from the mounted volume"
-  # Read the private key of L1 account for sending txs from peer
+if [ -n "$CERC_L1_ACCOUNTS_CSV_URL" ] && \
+  l1_accounts_response=$(curl -L --write-out '%{http_code}' --silent --output /dev/null "$CERC_L1_ACCOUNTS_CSV_URL") && \
+  [ "$l1_accounts_response" -eq 200 ];
+then
+  echo "Fetching L1 account credentials using provided URL"
+  mkdir -p /geth-accounts
+  wget -O /geth-accounts/accounts.csv "$CERC_L1_ACCOUNTS_CSV_URL"
+
+  # Read the private key of an L1 account for sending txs from peer
  CERC_PRIVATE_KEY_PEER=$(awk -F, 'NR==2{print $NF}' /geth-accounts/accounts.csv)
 else
-  echo "Using CERC_PRIVATE_KEY_PEER from env"
+  echo "Couldn't fetch L1 account credentials, using CERC_PRIVATE_KEY_PEER from env"
 fi

 # Read in the config template TOML file and modify it
--- a/app/data/container-build/cerc-mobymask/Dockerfile
+++ b/app/data/container-build/cerc-mobymask/Dockerfile
@ -1,6 +1,6 @@
 FROM node:16.17.1-alpine3.16

-RUN apk --update --no-cache add python3 alpine-sdk jq bash
+RUN apk --update --no-cache add python3 alpine-sdk jq bash curl wget

 WORKDIR /app

--- a/app/data/container-build/cerc-optimism-contracts/Dockerfile
+++ b/app/data/container-build/cerc-optimism-contracts/Dockerfile
@ -2,7 +2,7 @@ FROM cerc/foundry:local

 # Install node (local foundry is a debian based image)
 RUN apt-get update \
-  && apt-get install -y curl \
+  && apt-get install -y curl wget \
  && curl --silent --location https://deb.nodesource.com/setup_16.x | bash - \
  && apt-get update \
  && apt-get install -y nodejs git busybox jq \
--- a/app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile
+++ b/app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile
@ -1,7 +1,7 @@
 FROM ubuntu:22.04

 RUN apt-get update \
-  && apt-get install -y curl gnupg build-essential \
+  && apt-get install -y curl wget gnupg build-essential \
  && curl --silent --location https://deb.nodesource.com/setup_18.x | bash - \
  && apt-get update \
  && apt-get install -y nodejs git busybox jq \
--- a/app/data/stacks/fixturenet-eth/README.md
+++ b/app/data/stacks/fixturenet-eth/README.md
@ -117,8 +117,8 @@ Clear volumes created by this stack:

 ```bash
 # List all relevant volumes
-$ docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_bootnode_lighthouse_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data|.*fixturenet_geth_accounts"
+$ docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_bootnode_lighthouse_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data"

 # Remove all the listed volumes
-$ docker volume rm $(docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_bootnode_lighthouse_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data|.*fixturenet_geth_accounts")
+$ docker volume rm $(docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_bootnode_lighthouse_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data")
 ```
--- a/app/data/stacks/fixturenet-optimism/README.md
+++ b/app/data/stacks/fixturenet-optimism/README.md
@ -76,10 +76,10 @@ Clear volumes created by this stack:

 ```bash
 # List all relevant volumes
-docker volume ls -q --filter "name=.*fixturenet_geth_accounts|.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"
+docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"

 # Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=.*fixturenet_geth_accounts|.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
+docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
 ```

 ## Troubleshooting
--- a/app/data/stacks/fixturenet-optimism/l2-only.md
+++ b/app/data/stacks/fixturenet-optimism/l2-only.md
@ -46,8 +46,13 @@ Create and update an env file to be used in the next step ([defaults](../../conf
  CERC_L1_HOST=
  CERC_L1_PORT=

-  # Credentials for accounts on L1 to send balance to Optimism Proxy contract from
+  # URL to get CSV with credentials for accounts on L1
+  # that are used to send balance to Optimism Proxy contract
  # (enables them to do transactions on L2)
+  CERC_L1_ACCOUNTS_CSV_URL=
+
+  # OR
+  # Specify the required account credentials
  CERC_L1_ADDRESS=
  CERC_L1_PRIV_KEY=
  CERC_L1_ADDRESS_2=
@ -91,10 +96,10 @@ Clear volumes created by this stack:

 ```bash
 # List all relevant volumes
-docker volume ls -q --filter "name=.*fixturenet_geth_accounts|.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"
+docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"

 # Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=.*fixturenet_geth_accounts|.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
+docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
 ```

 ## Troubleshooting
--- a/app/data/stacks/mobymask-v2/README.md
+++ b/app/data/stacks/mobymask-v2/README.md
@ -114,8 +114,8 @@ Clear volumes created by this stack:

 ```bash
 # List all relevant volumes
-docker volume ls -q --filter "name=.*mobymask_watcher_db_data|.*peers_ids|.*mobymask_deployment|.*fixturenet_geth_accounts|.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"
+docker volume ls -q --filter "name=.*mobymask_watcher_db_data|.*peers_ids|.*mobymask_deployment|.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"

 # Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=.*mobymask_watcher_db_data|.*peers_ids|.*mobymask_deployment|.*fixturenet_geth_accounts|.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
+docker volume rm $(docker volume ls -q --filter "name=.*mobymask_watcher_db_data|.*peers_ids|.*mobymask_deployment|.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
 ```
--- a/app/data/stacks/mobymask-v2/mobymask-only.md
+++ b/app/data/stacks/mobymask-v2/mobymask-only.md
@ -49,7 +49,11 @@ Create and update an env file to be used in the next step ([defaults](../../conf
  CERC_L2_NODE_HOST=
  CERC_L2_NODE_PORT=

-  # Credentials for accounts to perform txs on L2
+  # URL to get CSV with credentials for accounts on L1 to perform txs on L2
+  CERC_L1_ACCOUNTS_CSV_URL=
+
+  # OR
+  # Specify the required account credentials
  CERC_PRIVATE_KEY_DEPLOYER=
  CERC_PRIVATE_KEY_PEER=

@ -118,8 +122,8 @@ Clear volumes created by this stack:

 ```bash
 # List all relevant volumes
-docker volume ls -q --filter "name=.*mobymask_watcher_db_data|.*peers_ids|.*mobymask_deployment|.*fixturenet_geth_accounts"
+docker volume ls -q --filter "name=.*mobymask_watcher_db_data|.*peers_ids|.*mobymask_deployment"

 # Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=.*mobymask_watcher_db_data|.*peers_ids|.*mobymask_deployment|.*fixturenet_geth_accounts")
+docker volume rm $(docker volume ls -q --filter "name=.*mobymask_watcher_db_data|.*peers_ids|.*mobymask_deployment")
 ```