Update README and condition for cleanup (#58)

* Update README and condition for cleanup

* Utilize config file for ipld-eth-beacon

* Update CICD to use config file on container build.

* Add env file for build cp

* Add a sleep before running the test

.github/workflows/generic-testing.yml

@@ -16,7 +16,7 @@ on:
         required: true
 
 env:
-  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '3048a224100ceb122d6da71328bf3803dff72a01' }}
+  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '7fb664270a0ba09e2caa3095e8c91f3fdb5b38af' }}
   ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref  || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
   ssz-data-ref: ${{ inputs.ssz-data-ref || 'main' }}
   GOPATH: /tmp/go
@@ -48,9 +48,7 @@ jobs:
         run: |
           echo vulcanize_ipld_eth_beacon_db=$GITHUB_WORKSPACE/ipld-eth-beacon-db/ > ./config.sh
           echo vulcanize_ipld_eth_beacon_indexer=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer >> ./config.sh
-          echo eth_beacon_capture_mode=boot >> ./config.sh
-          echo eth_beacon_skip_sync=true >> ./config.sh
-          echo eth_beacon_known_gap_increment=1000000 >> ./config.sh
+          echo eth_beacon_config_file=$GITHUB_WORKSPACE/ipld-eth-beacon-indexer/config/cicd/boot.ipld-eth-beacon-indexer.json >> ./config.sh
           cat ./config.sh
 
       - name: Run docker compose
@@ -65,7 +63,7 @@ jobs:
       - name: Check to make sure HEALTH file is present
         shell: bash
         run: |
-          until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" cp ipld-eth-beacon-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
+          until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" --env-file ./config.sh cp ipld-eth-beacon-indexer:/root/HEALTH ./HEALTH) ; do sleep 10; done
           cat ./HEALTH
           if [[ "$(cat ./HEALTH)" -eq "0" ]]; then echo "Application boot successful" && (exit 0); else docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-beacon-indexer.yml" cp ipld-eth-beacon-indexer:/root/ipld-eth-beacon-indexer.log . && cat ipld-eth-beacon-indexer.log && (exit 1); fi
 

.github/workflows/system-tests.yml

@@ -14,7 +14,7 @@ on:
       BC_ADDRESS:
         required: true
 env:
-  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '3048a224100ceb122d6da71328bf3803dff72a01' }}
+  stack-orchestrator-ref: ${{ inputs.stack-orchestrator-ref || '7fb664270a0ba09e2caa3095e8c91f3fdb5b38af' }}
   ipld-eth-beacon-db-ref: ${{ inputs.ipld-eth-beacon-db-ref  || '3dfe416302d553f8240f6051c08a7899b0e39e12' }}
   GOPATH: /tmp/go
   bc_protocol: "http"
@@ -81,11 +81,12 @@ jobs:
 
       - name: Run the tests using Make
         run: |
+          sleep 20
           cd ipld-eth-beacon-indexer
           make system-test-ci
 
       - name: Clean up the docker containers
-        if: steps.compose.outcome == 'success'
+        if: always() && steps.compose.outcome == 'success'
         shell: bash
         run: |
           /usr/local/bin/docker-compose \

.gitignore

@@ -6,4 +6,5 @@ cover.profile
 temp/*
 .vscode/*
 pkg/beaconclient/ssz-data/
-*.test
+*.test
+ipld-eth-beacon-indexer.log

README.md

@@ -27,7 +27,7 @@ To run the application, do as follows:
 
    ```
    ./wrapper.sh -e skip \
-   -d ../docker/local/docker-compose-eth-beacon-db.yml \
+   -d ../docker/local/docker-compose-ipld-eth-beacon-db.yml \
    -d ../docker/latest/docker-compose-lighthouse.yml \
    -v remove \
    -p ../local-config.sh
@@ -37,7 +37,7 @@ To run the application, do as follows:
 2. Run the start up command.
 
 ```
-go run -race main.go capture historic --config ./example.ipld-eth-beacon-indexer-config.json
+go run -race main.go capture full --config ./example.ipld-eth-beacon-indexer-config.json
 ```
 
 ## Running Tests

config/cicd/boot.ipld-eth-beacon-indexer.json

@@ -0,0 +1,40 @@
+{
+  "db": {
+    "address": "ipld-eth-beacon-db",
+    "password": "password",
+    "port": 5432,
+    "username": "vdbm",
+    "name": "vulcanize_testing",
+    "driver": "PGX"
+  },
+  "bc": {
+    "address": "lighthouse",
+    "port": 5052,
+    "type": "lighthouse",
+    "bootRetryInterval": 30,
+    "bootMaxRetry": 5,
+    "maxHistoricProcessWorker": 2,
+    "connectionProtocol": "http",
+    "uniqueNodeIdentifier": 100,
+    "checkDb": true
+  },
+  "t": {
+    "skipSync": true
+  },
+  "log": {
+    "level": "debug",
+    "output": true,
+    "file": "./ipld-eth-beacon-indexer.log",
+    "format": "json"
+  },
+  "kg": {
+    "increment": 10000,
+    "processKnownGaps": true,
+    "maxKnownGapsWorker": 2
+  },
+  "pm": {
+    "address": "localhost",
+    "port": 9000,
+    "metrics": true
+  }
+}

config/example.ipld-eth-beacon-indexer-config.json

@@ -8,7 +8,7 @@
     "driver": "PGX"
   },
   "bc": {
-    "address": "10.203.8.51",
+    "address": "localhost",
     "port": 5052,
     "type": "lighthouse",
     "bootRetryInterval": 30,

entrypoint.sh

@@ -3,30 +3,9 @@
 sleep 10
 echo "Starting ipld-eth-beacon-indexer"
 
-echo /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \
-  --db.password $DB_PASSWORD \
-  --db.port $DB_PORT \
-  --db.username $DB_USER \
-  --db.name $DB_NAME \
-  --db.driver $DB_DRIVER \
-  --bc.address $BC_ADDRESS \
-  --bc.port $BC_PORT \
-  --log.level $LOG_LEVEL\
-  --t.skipSync=$SKIP_SYNC \
-  --kg.increment $KNOWN_GAP_INCREMENT
-
-/root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --db.address $DB_ADDRESS \
-  --db.password $DB_PASSWORD \
-  --db.port $DB_PORT \
-  --db.username $DB_USER \
-  --db.name $DB_NAME \
-  --db.driver $DB_DRIVER \
-  --bc.address $BC_ADDRESS \
-  --bc.port $BC_PORT \
-  --log.level $LOG_LEVEL \
-  --t.skipSync=$SKIP_SYNC \
-  --kg.increment $KNOWN_GAP_INCREMENT
+echo /root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json
 
+/root/ipld-eth-beacon-indexer capture ${CAPTURE_MODE} --config /root/ipld-eth-beacon-config.json
 rv=$?
 
 if [ $rv != 0 ]; then