Add independent config for fixturenet-plugeth stack

Reviewed-on: #815
Co-authored-by: David Boreham <david@bozemanpass.com>
Co-committed-by: David Boreham <david@bozemanpass.com>

.gitea/workflows/triggers/test-external-stack

@@ -1 +1,2 @@
 Change this file to trigger running the external-stack CI job
+trigger

stack_orchestrator/data/compose/docker-compose-fixturenet-plugeth.yml

@@ -3,7 +3,7 @@ services:
     restart: always
     hostname: fixturenet-eth-bootnode-geth
     env_file:
-      - ../config/fixturenet-eth/fixturenet-eth.env
+      - ../config/fixturenet-plugeth/fixturenet-eth.env
     environment:
       RUN_BOOTNODE: "true"
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
@@ -24,8 +24,9 @@ services:
       CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect}
       CERC_STATEDIFF_DB_NODE_ID: 1
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ALLOW_UNPROTECTED_TXS: ${CERC_ALLOW_UNPROTECTED_TXS:-false}
     env_file:
-      - ../config/fixturenet-eth/fixturenet-eth.env
+      - ../config/fixturenet-plugeth/fixturenet-eth.env
     image: cerc/fixturenet-plugeth-plugeth:local
     volumes:
       - fixturenet_plugeth_geth_1_data:/root/ethdata
@@ -55,7 +56,7 @@ services:
     environment:
       CERC_KEEP_RUNNING_AFTER_GETH_EXIT: "true"
     env_file:
-      - ../config/fixturenet-eth/fixturenet-eth.env
+      - ../config/fixturenet-plugeth/fixturenet-eth.env
     image: cerc/fixturenet-plugeth-plugeth:local
     depends_on:
       - fixturenet-eth-bootnode-geth
@@ -82,7 +83,7 @@ services:
       retries: 10
       start_period: 30s
     env_file:
-      - ../config/fixturenet-eth/fixturenet-eth.env
+      - ../config/fixturenet-plugeth/fixturenet-eth.env
     environment:
       NODE_NUMBER: "1"
       ETH1_ENDPOINT: "http://fixturenet-eth-geth-1:8545"
@@ -108,7 +109,7 @@ services:
       retries: 10
       start_period: 30s
     env_file:
-      - ../config/fixturenet-eth/fixturenet-eth.env
+      - ../config/fixturenet-plugeth/fixturenet-eth.env
     environment:
       NODE_NUMBER: "2"
       ETH1_ENDPOINT: "http://fixturenet-eth-geth-2:8545"

stack_orchestrator/data/compose/docker-compose-graph-node.yml

@@ -16,8 +16,12 @@ services:
       postgres_pass: password
       postgres_db: graph-node
       ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
+      # Env varaibles reference: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
       GRAPH_LOG: debug
       ETHEREUM_REORG_THRESHOLD: 3
+      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
+      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
+      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
     entrypoint: ["bash", "-c"]
     # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
     command: |

stack_orchestrator/data/config/fixturenet-plugeth/fixturenet-eth.env

@@ -0,0 +1,28 @@
+# The password used to access test accounts (eg, via personal_unlockAccount).  The password is the same for all accounts.
+ACCOUNT_PASSWORD=secret1212
+
+# ENODE of the geth bootnode.
+BOOTNODE_KEY="b0ac22adcad37213c7c565810a50f1772291e7b0ce53fb73e7ec2a3c75bc13b5"
+ENODE="enode://af22c29c316ad069cf48a09a4ad5cf04a251b411e45098888d114c6dd7f489a13786620d5953738762afa13711d4ffb3b19aa5de772d8af72f851f7e9c5b164a@fixturenet-eth-bootnode-geth:30303"
+
+# JWT shared by geth and lighthouse for authentication.
+JWT="0x6cdcac3501046a08e186730dd8bd136cfaf0fdc1fc955f6e15ad3068c0ff2af0"
+
+# URL to download the ENR of the lighthouse bootnode (generated at first start).
+ENR_URL="http://fixturenet-eth-bootnode-lighthouse:3000/bootnode/enr.dat"
+
+# DB connection settings for statediffing (see docker-compose-db.yml)
+CERC_STATEDIFF_DB_HOST="ipld-eth-db"
+CERC_STATEDIFF_DB_PORT=5432
+CERC_STATEDIFF_DB_NAME="cerc_testing"
+CERC_STATEDIFF_DB_USER="vdbm"
+CERC_STATEDIFF_DB_PASSWORD="password"
+CERC_STATEDIFF_DB_GOOSE_MIN_VER=${CERC_STATEDIFF_DB_GOOSE_MIN_VER:-18}
+CERC_STATEDIFF_DB_LOG_STATEMENTS="${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false}"
+CERC_STATEDIFF_WORKERS=2
+
+CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"
+CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
+
+# Used by Lighthouse
+SECONDS_PER_ETH1_BLOCK=${SECONDS_PER_ETH1_BLOCK:-3}

stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -109,6 +109,10 @@ else
       OTHER_OPTS="--miner.threads=1"
     fi
 
+    if [ "$CERC_ALLOW_UNPROTECTED_TXS" == "true" ]; then
+      OTHER_OPTS+=" --rpc.allow-unprotected-txs"
+    fi
+
     $START_CMD \
       --datadir="${CERC_ETH_DATADIR}" \
       --bootnodes="${ENODE}" \

stack_orchestrator/data/stacks/fixturenet-plugeth/README.md

@@ -0,0 +1,19 @@
+# fixturenet-plugeth
+
+A variation of `fixturenet-eth` that uses `plugeth` instead of `go-ethereum`.
+
+See `stacks/fixturenet-eth/README.md` for more information.
+
+## Containers
+
+* cerc/lighthouse
+* cerc/fixturenet-eth-plugeth
+* cerc/fixturenet-eth-lighthouse
+
+## Deploy the stack
+
+```bash
+laconic-so --stack fixturenet-plugeth setup-repositories
+laconic-so --stack fixturenet-plugeth build-containers
+laconic-so --stack fixturenet-plugeth deploy up
+```

stack_orchestrator/data/stacks/fixturenet-plugeth/stack.yml

@@ -0,0 +1,17 @@
+version: "1.0"
+name: fixturenet-plugeth
+description: "Plugeth Ethereum Fixturenet"
+repos:
+  - git.vdb.to/cerc-io/plugeth@statediff
+  - git.vdb.to/cerc-io/plugeth-statediff
+  - git.vdb.to/cerc-io/lighthouse
+containers:
+  - cerc/plugeth-statediff
+  - cerc/plugeth
+  - cerc/fixturenet-eth-genesis
+  - cerc/fixturenet-plugeth-plugeth
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-lighthouse
+pods:
+  - fixturenet-plugeth

stack_orchestrator/data/stacks/graph-node/README.md

@@ -76,6 +76,19 @@ export ETH_RPC_PORT=
 # The etherum network(s) graph-node will connect to
 # Set this to a space-separated list of the networks where each entry has the form NAME:URL
 export ETH_NETWORKS=
+
+# Optional:
+
+# Timeout for ETH RPC requests in seconds (default: 180s)
+export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=
+
+# Number of times to retry ETH RPC requests (default: 10)
+export GRAPH_ETHEREUM_REQUEST_RETRIES=
+
+# Maximum number of blocks to scan for triggers in each request (default: 2000)
+export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=
+
+# Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
 ```
 
 Example env file:
@@ -85,6 +98,10 @@ export ETH_RPC_HOST=filecoin.chainup.net
 export ETH_RPC_PORT=443
 
 export ETH_NETWORKS=filecoin:https://filecoin.chainup.net/rpc/v1
+
+export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=360
+export GRAPH_ETHEREUM_REQUEST_RETRIES=5
+export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
 ```
 
 Set the environment variables:

stack_orchestrator/deploy/deployment_create.py

@@ -26,7 +26,8 @@ from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                      global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
-                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file)
+                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file,
+                                     resolve_config_dir)
 from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
@@ -480,7 +481,6 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
     os.mkdir(destination_compose_dir)
     destination_pods_dir = deployment_dir_path.joinpath("pods")
     os.mkdir(destination_pods_dir)
-    data_dir = Path(__file__).absolute().parent.parent.joinpath("data")
     yaml = get_yaml()
     for pod in pods:
         pod_file_path = get_pod_file_path(stack_name, parsed_stack, pod)
@@ -497,7 +497,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
         config_dirs = {pod}
         config_dirs = config_dirs.union(extra_config_dirs)
         for config_dir in config_dirs:
-            source_config_dir = data_dir.joinpath("config", config_dir)
+            source_config_dir = resolve_config_dir(stack_name, config_dir)
             if os.path.exists(source_config_dir):
                 destination_config_dir = deployment_dir_path.joinpath("config", config_dir)
                 # If the same config dir appears in multiple pods, it may already have been copied

stack_orchestrator/util.py

@@ -94,6 +94,20 @@ def get_plugin_code_paths(stack) -> List[Path]:
     return list(result)
 
 
+# # Find a config directory, looking first in any external stack
+# and if not found there, internally
+def resolve_config_dir(stack, config_dir_name: str):
+    if stack_is_external(stack):
+        # First try looking in the external stack for the compose file
+        config_base = Path(stack).parent.parent.joinpath("config")
+        proposed_dir = config_base.joinpath(config_dir_name)
+        if proposed_dir.exists():
+            return proposed_dir
+        # If we don't find it fall through to the internal case
+    config_base = get_internal_config_dir()
+    return config_base.joinpath(config_dir_name)
+
+
 # Find a compose file, looking first in any external stack
 # and if not found there, internally
 def resolve_compose_file(stack, pod_name: str):
@@ -153,7 +167,7 @@ def get_internal_compose_file_dir():
     return source_compose_dir
 
 
-def get_config_file_dir():
+def get_internal_config_dir():
     # TODO: refactor to use common code with deploy command
     data_dir = Path(__file__).absolute().parent.joinpath("data")
     source_config_dir = data_dir.joinpath("config")

tests/external-stack/run-test.sh

@@ -31,7 +31,7 @@ rm -rf $CERC_REPO_BASE_DIR
 mkdir -p $CERC_REPO_BASE_DIR
 # Clone the external test stack
 $TEST_TARGET_SO fetch-stack git.vdb.to/cerc-io/test-external-stack
-stack_name="$CERC_REPO_BASE_DIR/test-external-stack/stacks/test-external-stack"
+stack_name="$CERC_REPO_BASE_DIR/test-external-stack/stack-orchestrator/stacks/test-external-stack"
 TEST_TARGET_SO_STACK="$TEST_TARGET_SO --stack ${stack_name}"
 # Test bringing the test container up and down
 # with and without volume removal