Merge main

2023-07-18 18:24:46 -06:00 · 2023-07-18 18:24:46 -06:00 · 0f218ab3ed
commit 0f218ab3ed
parent dcfd7e3d45 c8b4f89335
15 changed files with 76 additions and 78 deletions
--- a/app/data/compose/docker-compose-fixturenet-optimism.yml
+++ b/app/data/compose/docker-compose-fixturenet-optimism.yml
@ -23,7 +23,7 @@ services:
    command: |
      "./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./run.sh"
    volumes:
-      - ../config/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh
+      - ../config/network/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh
      - ../container-build/cerc-optimism-contracts/hardhat-tasks/verify-contract-deployment.ts:/app/packages/contracts-bedrock/tasks/verify-contract-deployment.ts
      - ../container-build/cerc-optimism-contracts/hardhat-tasks/rekey-json.ts:/app/packages/contracts-bedrock/tasks/rekey-json.ts
      - ../container-build/cerc-optimism-contracts/hardhat-tasks/send-balance.ts:/app/packages/contracts-bedrock/tasks/send-balance.ts
@ -120,7 +120,7 @@ services:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
      CERC_L1_RPC: ${CERC_L1_RPC}
    volumes:
-      - ../config/wait-for-it.sh:/wait-for-it.sh
+      - ../config/network/wait-for-it.sh:/wait-for-it.sh
      - ../config/fixturenet-optimism/run-op-batcher.sh:/run-op-batcher.sh
      - l2_accounts:/l2-accounts:ro
    entrypoint: ["sh", "-c"]
@ -145,7 +145,7 @@ services:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
      CERC_L1_RPC: ${CERC_L1_RPC}
    volumes:
-      - ../config/wait-for-it.sh:/wait-for-it.sh
+      - ../config/network/wait-for-it.sh:/wait-for-it.sh
      - ../config/fixturenet-optimism/run-op-proposer.sh:/run-op-proposer.sh
      - l1_deployment:/contracts-bedrock:ro
      - l2_accounts:/l2-accounts:ro
--- a/app/data/compose/docker-compose-fixturenet-plugeth.yml
+++ b/app/data/compose/docker-compose-fixturenet-plugeth.yml
@ -29,7 +29,6 @@ services:
    image: cerc/fixturenet-plugeth-plugeth:local
    volumes:
      - fixturenet_plugeth_geth_1_data:/root/ethdata
-      - ../config/fixturenet-plugeth/plugins:/root/ethdata/plugins
    healthcheck:
      test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:8545/"]
      interval: 30s
@ -61,7 +60,6 @@ services:
      - fixturenet-eth-bootnode-geth
    volumes:
      - fixturenet_plugeth_geth_2_data:/root/ethdata
-      - ../config/fixturenet-plugeth/plugins:/root/ethdata/plugins

  fixturenet-eth-bootnode-lighthouse:
    restart: always
--- a/app/data/compose/docker-compose-mobymask-app.yml
+++ b/app/data/compose/docker-compose-mobymask-app.yml
@ -18,7 +18,7 @@ services:
    working_dir: /scripts
    command: ["sh", "mobymask-app-start.sh"]
    volumes:
-      - ../config/wait-for-it.sh:/scripts/wait-for-it.sh
+      - ../config/network/wait-for-it.sh:/scripts/wait-for-it.sh
      - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
      - peers_ids:/peers
      - mobymask_deployment:/server
@ -50,7 +50,7 @@ services:
    working_dir: /scripts
    command: ["sh", "mobymask-app-start.sh"]
    volumes:
-      - ../config/wait-for-it.sh:/scripts/wait-for-it.sh
+      - ../config/network/wait-for-it.sh:/scripts/wait-for-it.sh
      - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
      - peers_ids:/peers
      - mobymask_deployment:/server
--- a/app/data/compose/docker-compose-peer-test-app.yml
+++ b/app/data/compose/docker-compose-peer-test-app.yml
@ -14,7 +14,7 @@ services:
      CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
    command: ["sh", "test-app-start.sh"]
    volumes:
-      - ../config/wait-for-it.sh:/scripts/wait-for-it.sh
+      - ../config/network/wait-for-it.sh:/scripts/wait-for-it.sh
      - ../config/watcher-mobymask-v2/test-app-start.sh:/scripts/test-app-start.sh
      - peers_ids:/peers
    ports:
--- a/app/data/compose/docker-compose-watcher-mobymask-v2.yml
+++ b/app/data/compose/docker-compose-watcher-mobymask-v2.yml
@ -44,7 +44,7 @@ services:
      CERC_L2_NODE_PORT: ${CERC_L2_NODE_PORT}
    command: ["sh", "deploy-and-generate-invite.sh"]
    volumes:
-      - ../config/wait-for-it.sh:/app/packages/server/wait-for-it.sh
+      - ../config/network/wait-for-it.sh:/app/packages/server/wait-for-it.sh
      - ../config/watcher-mobymask-v2/secrets-template.json:/app/packages/server/secrets-template.json
      - ../config/watcher-mobymask-v2/deploy-and-generate-invite.sh:/app/packages/server/deploy-and-generate-invite.sh
      - mobymask_deployment:/app/packages/server
--- a/app/data/config/network/wait-for-it.sh
+++ b/app/data/config/network/wait-for-it.sh
--- a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh
+++ b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh
@ -7,11 +7,12 @@ fi
 ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
 NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
 NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
+CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}"
+CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}"

-HOME_DIR=`pwd`
 cd /opt/testnet/build/el
 python3 -m http.server 9898 &
-cd $HOME_DIR
+cd $HOME

 START_CMD="geth"
 if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
@ -34,7 +35,7 @@ trap 'cleanup' SIGINT SIGTERM

 if [ "true" == "$RUN_BOOTNODE" ]; then
    $START_CMD \
-       --datadir=~/ethdata \
+      --datadir="${CERC_ETH_DATADIR}" \
      --nodekeyhex="${BOOTNODE_KEY}" \
      --nodiscover \
      --ipcdisable \
@ -82,7 +83,7 @@ else
          fi
        fi
      done
-      STATEDIFF_OPTS="--statediff=true \
+      STATEDIFF_OPTS="--statediff \
      --statediff.db.host=$CERC_STATEDIFF_DB_HOST \
      --statediff.db.name=$CERC_STATEDIFF_DB_NAME \
      --statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \
@ -94,10 +95,15 @@ else
      --statediff.waitforsync=true \
      --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \
      --statediff.writing=true"
+
+      if [ -d "${CERC_PLUGINS_DIR}" ]; then
+        # With plugeth, we separate the statediff options by prefixing with ' -- '
+        STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}"
+      fi
    fi

    $START_CMD \
-      --datadir=~/ethdata \
+      --datadir="${CERC_ETH_DATADIR}" \
      --bootnodes="${ENODE}" \
      --allow-insecure-unlock \
      --http \
@ -124,8 +130,9 @@ else
      --metrics \
      --metrics.addr="0.0.0.0" \
      --verbosity=${CERC_GETH_VERBOSITY:-3} \
-      --vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
-      --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS} \
+      --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
+      --miner.etherbase="${ETHERBASE}" \
+      ${STATEDIFF_OPTS} \
      &

    geth_pid=$!
--- a/app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile
+++ b/app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile
@ -1,27 +0,0 @@
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
-
-FROM golang:1.19.4-bullseye AS delve
-RUN go install github.com/go-delve/delve/cmd/dlv@latest
-
-FROM ubuntu:22.04
-RUN apt-get update && \
-      apt-get install -y --no-install-recommends \
-        python3 python3-dev python3-pip curl wget jq gettext gettext-base openssl bash dnsutils postgresql-client make iproute2 netcat && \
-      rm -rf /var/lib/apt/lists/*
-
-COPY --from=delve  /go/bin/dlv /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
-COPY --from=ethgen /apps /apps
-
-RUN wget -O /usr/local/bin/geth https://github.com/openrelayxyz/plugeth/releases/download/v1.11.6.1.0/geth-linux-amd64-v1.1.0-v1.11.6.1.0 && chmod a+x /usr/local/bin/geth
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
-
-COPY genesis /opt/testnet
-COPY run-el.sh /opt/testnet/run.sh
-
-RUN cd /opt/testnet && make genesis-el
-
-RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
-
-ENTRYPOINT ["/opt/testnet/run.sh"]
--- a/app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh
+++ b/app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh
@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-# Build cerc/fixturenet-eth-plugeth
-set -x
-
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-if [ ! -d "${SCRIPT_DIR}/genesis" ]; then
-  cp -frp ${SCRIPT_DIR}/../cerc-fixturenet-eth-geth/genesis ${SCRIPT_DIR}/genesis
-fi
-
-if [ ! -d "${SCRIPT_DIR}/run-el.sh" ]; then
-  cp -fp ${SCRIPT_DIR}/../cerc-fixturenet-eth-geth/run-el.sh ${SCRIPT_DIR}/
-fi
-
-docker build -t cerc/fixturenet-eth-plugeth:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
--- a/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile
+++ b/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile
@ -1,20 +1,19 @@
 FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen

-FROM golang:1.19.4-bullseye AS delve
+FROM golang:1.19-alpine as delve
 RUN go install github.com/go-delve/delve/cmd/dlv@latest

-FROM ubuntu:22.04
-RUN apt-get update && \
-      apt-get install -y --no-install-recommends \
-        python3 python3-dev python3-pip curl wget jq gettext gettext-base openssl bash dnsutils postgresql-client make iproute2 netcat psmisc && \
-      rm -rf /var/lib/apt/lists/*
+FROM cerc/plugeth:local as geth
+FROM cerc/plugeth-statediff:local as statediff
+
+FROM alpine:3.17
+RUN apk add --no-cache python3 python3-dev py3-pip curl wget jq build-base gettext libintl openssl bash bind-tools postgresql-client

 COPY --from=delve  /go/bin/dlv /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps

-RUN wget -O /usr/local/bin/geth https://github.com/openrelayxyz/plugeth/releases/download/v1.11.6.1.0/geth-linux-amd64-v1.1.0-v1.11.6.1.0 && chmod a+x /usr/local/bin/geth
 RUN cd /apps/el-gen && pip3 install -r requirements.txt

 COPY genesis /opt/testnet
@ -22,7 +21,10 @@ COPY run-el.sh /opt/testnet/run.sh

 RUN cd /opt/testnet && make genesis-el

-RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
+COPY --from=geth  /usr/local/bin/geth /usr/local/bin/
+
+RUN mkdir -p /usr/local/lib/plugeth/
+COPY --from=statediff  /usr/local/lib/statediff.so /usr/local/lib/plugeth/

 # Snag the genesis block info.
 RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey
--- a/app/data/container-build/cerc-go-opera/build.sh
+++ b/app/data/container-build/cerc-go-opera/build.sh
@ -2,10 +2,6 @@
 # Build cerc/go-opera
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh

-# Checkout appropriate release; refer to https://docs.fantom.foundation/
-OPERA_TAG=${OPERA_TAG:-release/1.1.2-rc.5}
-git -C ${CERC_REPO_BASE_DIR}/go-opera checkout ${OPERA_TAG}
-
 # Repo's dockerfile gives build error because it's hardcoded for go 1.17; go 1.19 is required
 sed -i 's/FROM golang:1\.[0-9]*-alpine as builder/FROM golang:1.19-alpine as builder/' ${CERC_REPO_BASE_DIR}/go-opera/docker/Dockerfile.opera

--- a/app/data/stacks/fixturenet-plugeth-tx/stack.yml
+++ b/app/data/stacks/fixturenet-plugeth-tx/stack.yml
@ -5,14 +5,24 @@ repos:
  - github.com/cerc-io/tx-spammer
  - github.com/dboreham/foundry
  - github.com/cerc-io/lighthouse
+  - github.com/cerc-io/ipld-eth-db@v5
+  - github.com/cerc-io/ipld-eth-server@v5
+  - git.vdb.to/cerc-io/plugeth@statediff-wip
+  - git.vdb.to/cerc-io/plugeth-statediff@dev-local-build
 containers:
  - cerc/lighthouse
  - cerc/lighthouse-cli
+  - cerc/plugeth-statediff
+  - cerc/plugeth
  - cerc/fixturenet-plugeth-plugeth
  - cerc/fixturenet-plugeth-lighthouse
  - cerc/tx-spammer
  - cerc/foundry
+  - cerc/ipld-eth-db
+  - cerc/ipld-eth-server
 pods:
+  - ipld-eth-db
+  - ipld-eth-server
  - fixturenet-plugeth
  - foundry
  - tx-spammer
--- a/app/data/stacks/mainnet-go-opera/stack.yml
+++ b/app/data/stacks/mainnet-go-opera/stack.yml
@ -2,7 +2,7 @@ version: "1.1"
 name: mainnet-opera
 decription: "Fantom mainnet node"
 repos:
-  - github.com/Fantom-foundation/go-opera
+  - github.com/Fantom-foundation/go-opera@release/1.1.2-rc.5
 containers:
  - cerc/go-opera
 pods:
--- a/app/deployment.py
+++ b/app/deployment.py
@ -73,13 +73,14 @@ def start(ctx, extra_args):


@command.command()
+@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes")
@click.argument('extra_args', nargs=-1)  # help: command: down <service1> <service2>
@click.pass_context
-def down(ctx, extra_args):
+def down(ctx, delete_volumes, extra_args):
    # Get the stack config file name
    # TODO: add cluster name and env file here
    ctx.obj = make_deploy_context(ctx)
-    down_operation(ctx, extra_args, None)
+    down_operation(ctx, delete_volumes, extra_args)


 # stop is the preferred alias for down
--- a/app/deployment_create.py
+++ b/app/deployment_create.py
@ -111,6 +111,24 @@ def call_stack_deploy_setup(stack):
    return imported_stack.setup(None)


+# Inspect the pod yaml to find config files referenced in subdirectories
+# other than the one associated with the pod
+def _find_extra_config_dirs(parsed_pod_file, pod):
+    config_dirs = set()
+    services = parsed_pod_file["services"]
+    for service in services:
+        service_info = services[service]
+        if "volumes" in service_info:
+            for volume in service_info["volumes"]:
+                if ":" in volume:
+                    host_path = volume.split(":")[0]
+                    if host_path.startswith("../config"):
+                        config_dir = host_path.split("/")[2]
+                        if config_dir != pod:
+                            config_dirs.add(config_dir)
+    return config_dirs
+
+
@click.command()
@click.option("--output", required=True, help="Write yaml spec file here")
@click.pass_context
@ -163,13 +181,22 @@ def create(ctx, spec_file, deployment_dir):
    for pod in pods:
        pod_file_path = os.path.join(_get_compose_file_dir(), f"docker-compose-{pod}.yml")
        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
+        extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
+        if global_options(ctx).debug:
+            print(f"extra config dirs: {extra_config_dirs}")
        _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
        with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file:
            yaml.dump(parsed_pod_file, output_file)
        # Copy the config files for the pod, if any
-        source_config_dir = data_dir.joinpath("config", pod)
+        config_dirs = {pod}
+        config_dirs = config_dirs.union(extra_config_dirs)
+        for config_dir in config_dirs:
+            source_config_dir = data_dir.joinpath("config", config_dir)
            if os.path.exists(source_config_dir):
-            copytree(source_config_dir, os.path.join(deployment_dir, "config", pod))
+                destination_config_dir = os.path.join(deployment_dir, "config", config_dir)
+                # If the same config dir appears in multiple pods, it may already have been copied
+                if not os.path.exists(destination_config_dir):
+                    copytree(source_config_dir, destination_config_dir)


@click.command()
@ -182,3 +209,4 @@ def create(ctx, spec_file, deployment_dir):
 def setup(ctx, node_moniker, key_name, initialize_network, join_network, create_network):
    stack = global_options(ctx).stack
    call_stack_deploy_setup(stack)
+