Merge pull request #344 from cerc-io/console-docs

document the laconicd / sdk/ registry CLI / web console Stack

Former-commit-id: c712c181fc

app/data/compose/docker-compose-fixturenet-eth.yml

@@ -49,6 +49,8 @@ services:
       timeout: 10s
       retries: 10
       start_period: 3s
+    environment:
+      CERC_KEEP_RUNNING_AFTER_GETH_EXIT: "true"
     env_file:
       - ../config/fixturenet-eth/fixturenet-eth.env
     image: cerc/fixturenet-eth-geth:local

app/data/compose/docker-compose-fixturenet-optimism.yml

@@ -122,6 +122,8 @@ services:
     # Waits for L1 endpoint to be up before running the batcher
     command: |
       "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-batcher.sh"
+    ports:
+      - "127.0.0.1:8548:8548"
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -145,6 +147,8 @@ services:
     # Waits for L1 endpoint to be up before running the proposer
     command: |
       "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-proposer.sh"
+    ports:
+      - "127.0.0.1:8560:8560"
     extra_hosts:
       - "host.docker.internal:host-gateway"
 

app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -127,3 +127,9 @@ else
 fi
 
 wait $geth_pid
+
+if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then
+  while [ 1 -eq 1 ]; do
+    sleep 60
+  done
+fi

app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Exports the complete fixturenet-eth ethdb data to a tarball (default, ./ethdb.tgz), waiting for a minimum
+# block height (default 1000) to be reached before exporting.
+
+# Usage: export-ethdb.sh [min_block_number=1000] [output_file=./ethdb.tgz]
+
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+GETH_EXPORT_MIN_BLOCK=${1:-${GETH_EXPORT_MIN_BLOCK:-1000}}
+
+# Wait for block.
+${SCRIPT_DIR}/status.sh $GETH_EXPORT_MIN_BLOCK
+if [[ $? -ne 0 ]]; then
+  echo "Unable to export ethdb." 1>&2
+  exit 1
+fi
+
+# Stop geth.
+echo -n "Exporting ethdb.... "
+GETH_CONTAINER=`docker ps -q -f "name=${CERC_SO_COMPOSE_PROJECT}-fixturenet-eth-geth-2-1"`
+if [[ -z "$GETH_CONTAINER" ]]; then
+  echo "not found"
+  exit 1
+fi
+
+docker exec $GETH_CONTAINER sh -c 'rm -rf /root/tmp && mkdir -p /root/tmp/export'
+docker exec $GETH_CONTAINER sh -c 'ln -s /opt/testnet/build/el/geth.json /root/tmp/export/genesis.json && ln -s /root/ethdata /root/tmp/export/'
+docker exec $GETH_CONTAINER sh -c 'cat /root/tmp/export/genesis.json | jq ".config" > /root/tmp/export/genesis.config.json'
+
+# Stop geth and zip up ethdb.
+docker exec $GETH_CONTAINER sh -c 'curl -s --location "localhost:8545" --header "Content-Type: application/json" --data "{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"eth_getBlockByNumber\", \"params\": [\"0x0\", false]}" > /root/tmp/export/eth_getBlockByNumber_0x0.json'
+docker exec $GETH_CONTAINER sh -c 'curl -s --location "localhost:8545" --header "Content-Type: application/json" --data "{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"eth_blockNumber\", \"params\": []}" > /root/tmp/export/eth_blockNumber.json'
+docker exec $GETH_CONTAINER sh -c "killall geth && sleep 2 && tar chzf /root/tmp/ethdb.tgz -C /root/tmp/export ."
+
+# Copy ethdb to host.
+GETH_EXPORT_FILE=${2:-${GETH_EXPORT_FILE:-./ethdb.tgz}}
+docker cp $GETH_CONTAINER:/root/tmp/ethdb.tgz $GETH_EXPORT_FILE
+echo "$GETH_EXPORT_FILE"
+docker exec $GETH_CONTAINER sh -c "rm -rf /root/tmp"
+
+# Restart the container to get geth back up and running.
+docker restart $GETH_CONTAINER >/dev/null

app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh

@@ -2,9 +2,10 @@
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
-STATUSES=("geth to generate DAG" "beacon phase0" "beacon altair" "beacon bellatrix pre-merge" "beacon bellatrix merge")
-STATUS=0
 
+MIN_BLOCK_NUM=${1:-${MIN_BLOCK_NUM:-3}}
+STATUSES=("geth to generate DAG" "beacon phase0" "beacon altair" "beacon bellatrix pre-merge" "beacon bellatrix merge" "block number $MIN_BLOCK_NUM")
+STATUS=0
 
 LIGHTHOUSE_BASE_URL=${LIGHTHOUSE_BASE_URL}
 GETH_BASE_URL=${GETH_BASE_URL}
@@ -13,18 +14,29 @@ GETH_BASE_URL=${GETH_BASE_URL}
 # or some execution environment-neutral mechanism.
 if [ -z "$LIGHTHOUSE_BASE_URL" ]; then
   LIGHTHOUSE_CONTAINER=`docker ps -q -f "name=fixturenet-eth-lighthouse-1-1"`
+  if [ -z "$LIGHTHOUSE_CONTAINER" ]; then
+    echo "Lighthouse container not found." 1>&2
+    exit 1
+  fi
   LIGHTHOUSE_PORT=`docker port $LIGHTHOUSE_CONTAINER 8001 | cut -d':' -f2`
   LIGHTHOUSE_BASE_URL="http://localhost:${LIGHTHOUSE_PORT}"
 fi
 
 if [ -z "$GETH_BASE_URL" ]; then
   GETH_CONTAINER=`docker ps -q -f "name=fixturenet-eth-geth-1-1"`
+  if [ -z "$GETH_CONTAINER" ]; then
+    echo "Lighthouse container not found." 1>&2
+    exit 1
+  fi
   GETH_PORT=`docker port $GETH_CONTAINER 8545 | cut -d':' -f2`
   GETH_BASE_URL="http://localhost:${GETH_PORT}"
 fi
 
+MARKER="."
+
 function inc_status() {
   echo " done"
+  MARKEr="."
   STATUS=$((STATUS + 1))
   if [ $STATUS -lt ${#STATUSES[@]} ]; then
     echo -n "Waiting for ${STATUSES[$STATUS]}..."
@@ -34,7 +46,7 @@ function inc_status() {
 echo -n "Waiting for ${STATUSES[$STATUS]}..."
 while [ $STATUS -lt ${#STATUSES[@]} ]; do
   sleep 1
-  echo -n "."
+  echo -n "$MARKER"
   case $STATUS in
     0)
       result=`wget --no-check-certificate --quiet -O - --method POST --header 'Content-Type: application/json' \
@@ -67,5 +79,13 @@ while [ $STATUS -lt ${#STATUSES[@]} ]; do
         inc_status
       fi
       ;;
+    5)
+      result=`wget --no-check-certificate --quiet -O - "$LIGHTHOUSE_BASE_URL/eth/v2/beacon/blocks/head" | jq -r '.data.message.body.execution_payload.block_number'`
+      if [ ! -z "$result" ] && [ $result -gt $MIN_BLOCK_NUM ]; then
+        inc_status
+      else
+        MARKER="$result "
+      fi
+      ;;
   esac
 done

app/data/stacks/build-support/stack.yml

@@ -1,6 +1,5 @@
-version: "1.1"
+version: "1.2"
 name: build-support
 decription: "Build Support Components"
 containers:
   - cerc/builder-js
-  - cerc/builder-gerbil

app/data/stacks/fixturenet-eth-tx/README.md

@@ -0,0 +1,37 @@
+# fixturenet-eth-tx
+
+A variation of `fixturenet-eth` that automatically generates transactions using `tx-spammer`.
+
+See `stacks/fixturenet-eth/README.md` for more information.
+
+## Containers
+
+* cerc/go-ethereum
+* cerc/lighthouse
+* cerc/fixturenet-eth-geth
+* cerc/fixturenet-eth-lighthouse
+* cerc/tx-spammer
+
+## Deploy the stack
+```
+$ laconic-so --stack fixturenet-eth-tx setup-repositories
+$ laconic-so --stack fixturenet-eth-tx build-containers
+$ laconic-so --stack fixturenet-eth-tx deploy up
+```
+
+## Export the ethdb (optional)
+
+It is easy to export data from the fixturenet for offline processing of the raw ethdb files (eg, by eth-statediff-service) using the `export-ethdb.sh` script.
+
+For example:
+
+```
+$ app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh 500
+Waiting for geth to generate DAG.... done
+Waiting for beacon phase0.... done
+Waiting for beacon altair.... done
+Waiting for beacon bellatrix pre-merge.... done
+Waiting for beacon bellatrix merge.... done
+Waiting for block number 500.... done
+Exporting ethdb.... ./ethdb.tgz
+```

app/data/stacks/fixturenet-eth-tx/stack.yml

@@ -0,0 +1,18 @@
+version: "1.2"
+name: fixturenet-eth-tx
+decription: "Ethereum Fixturenet w/ tx-spammer"
+repos:
+  - cerc-io/go-ethereum
+  - cerc-io/tx-spammer
+  - dboreham/foundry
+containers:
+  - cerc/go-ethereum
+  - cerc/lighthouse
+  - cerc/fixturenet-eth-geth
+  - cerc/fixturenet-eth-lighthouse
+  - cerc/tx-spammer
+  - cerc/foundry
+pods:
+  - fixturenet-eth
+  - foundry
+  - tx-spammer

app/data/stacks/fixturenet-optimism/README.md

@@ -2,7 +2,7 @@
 
 Instructions to setup and deploy an end-to-end L1+L2 stack with [fixturenet-eth](../fixturenet-eth/) (L1) and [Optimism](https://stack.optimism.io) (L2)
 
-We support running just the L2 part of stack, given an external L1 endpoint. Follow [l2-only](./l2-only.md) for the same.
+We support running just the L2 part of stack, given an external L1 endpoint. Follow the [L2 only doc](./l2-only.md) for the same.
 
 ## Setup
 
@@ -28,6 +28,8 @@ Build the container images:
 laconic-so --stack fixturenet-optimism build-containers
 ```
 
+Note: this will take >10 mins depending on the specs of your machine, and **requires** 16GB of memory or greater.
+
 This should create the required docker images in the local image registry:
 * `cerc/go-ethereum`
 * `cerc/lighthouse`
@@ -48,12 +50,14 @@ Deploy the stack:
 laconic-so --stack fixturenet-optimism deploy up
 ```
 
-The `fixturenet-optimism-contracts` service may take a while (`~15 mins`) to complete running as it:
+If you get the error `service "fixturenet-optimism-contracts" didn't complete successfully: exit 1` with ~25 lines of Traceback, wait 15-20 mins then re-run the command.
+
+The `fixturenet-optimism-contracts` service takes a while to complete running as it:
 1. waits for the 'Merge' to happen on L1
 2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups)
 3. deploys the L1 contracts
 
-To list down and monitor the running containers:
+To list and monitor the running containers:
 
 ```bash
 laconic-so --stack fixturenet-optimism deploy ps

app/data/stacks/mobymask-v2/watcher-p2p-network/web-app.md

@@ -48,7 +48,7 @@ Add the following contents to `mobymask-app.env`:
 
   # DO NOT CHANGE THESE VALUES
   CERC_DEPLOYED_CONTRACT="0x2B6AFbd4F479cE4101Df722cF4E05F941523EaD9"
-  CERC_RELAY_NODES=["/dns4/relay1.dev.vdb.to/tcp/443/wss/p2p/12D3KooWAx83SM9GWVPc9v9fNzLzftRX6EaAFMjhYiFxRYqctcW1", "/dns4/relay2.dev.vdb.to/tcp/443/wss/p2p/12D3KooWBycy6vHVEfUwwYRbPLBdb5gx9gtFSEMpErYPUjUkDNkm", "/dns4/relay3.dev.vdb.to/tcp/443/wss/p2p/12D3KooWARcUJsiGCgiygiRVVK94U8BNSy8DFBbzAF3B6orrabwn"]
+  CERC_RELAY_NODES=["/dns4/relay1.dev.vdb.to/tcp/443/wss/p2p/12D3KooWAx83SM9GWVPc9v9fNzLzftRX6EaAFMjhYiFxRYqctcW1","/dns4/relay2.dev.vdb.to/tcp/443/wss/p2p/12D3KooWBycy6vHVEfUwwYRbPLBdb5gx9gtFSEMpErYPUjUkDNkm","/dns4/relay3.dev.vdb.to/tcp/443/wss/p2p/12D3KooWARcUJsiGCgiygiRVVK94U8BNSy8DFBbzAF3B6orrabwn"]
   ```
 
 Replace `CERC_APP_WATCHER_URL` with the watcher's endpoint (eg. `https://mobymask.example.com`)

app/data/stacks/mobymask-v2/web-apps.md

@@ -22,7 +22,7 @@ Create and update an env file to be used in the next step ([defaults](../../conf
 
   ```bash
   # Set of relay nodes to be used by the web-app
-  # (use double quotes " for strings)
+  # (use double quotes " for strings, avoid space after commas)
   # Eg. CERC_RELAY_NODES=["/dns4/example.com/tcp/443/wss/p2p/12D3KooWGHmDDCc93XUWL16FMcTPCGu2zFaMkf67k8HZ4gdQbRDr"]
   CERC_RELAY_NODES=[]
 

docs/laconicd-fixturenet.md

@@ -0,0 +1,183 @@
+# Running a laconicd fixturenet with console
+
+The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following 4 repositories had to be cloned and setup manually:
+
+- https://github.com/cerc-io/laconicd
+- https://github.com/cerc-io/laconic-sdk
+- https://github.com/cerc-io/laconic-registry-cli
+- https://github.com/cerc-io/laconic-console
+
+Now, with Stack Orchestrator, it is a few quick commands. Additionally, the `docker` and `docker compose` integration on the back-end allows the stack to easily persist, facilitating workflows.
+
+## Setup laconic-so
+
+To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the user experience, this tutorial is focused on using a fresh Digital Ocean (DO) droplet with similar specs: 
+16 GB Memory / 8 Intel vCPUs / 160 GB Disk.
+
+1. Login to the droplet as root (either by SSH key or password set in the DO console)
+
+```
+ssh root@IP
+```
+
+2. Get the install script, give it executable permissions, and run it:
+
+```
+curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-ubuntu.sh
+```
+```
+chmod +x install.sh
+```
+```
+bash install.sh
+```
+
+3. Confirm docker was installed and activate the changes in `~/.profile`:
+
+```
+docker run hello-world
+```
+```
+source ~/.profile
+```
+
+4. Verify installation:
+
+```
+laconic-so version
+```
+
+## Setup the laconic fixturenet stack
+
+1. Get the repositories
+
+```
+laconic-so --stack fixturenet-laconic-loaded setup-repositories --include cerc-io/laconicd,cerc-io/laconic-sdk,cerc-io/laconic-registry-cli,cerc-io/laconic-console
+```
+
+2. Set this environment variable to the Laconic self-hosted Gitea instance:
+
+```
+export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
+```
+
+3. Build the containers:
+
+```
+laconic-so --stack fixturenet-laconic-loaded build-containers
+```
+
+It's possible to run into an `ESOCKETTIMEDOUT` error, e.g., `error An unexpected error occurred: "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz: ESOCKETTIMEDOUT"`. This may happen even if you have a great internet connection. In that case, re-run the `build-containers` command.
+
+4. Set this environment variable to your droplet's IP address:
+
+```
+export LACONIC_HOSTED_ENDPOINT=http://<your-IP>
+```
+
+5. Deploy the stack:
+
+```
+laconic-so --stack fixturenet-laconic-loaded deploy up
+```
+
+6. Check the logs:
+
+```
+laconic-so --stack fixturenet-laconic-loaded deploy logs
+```
+
+You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
+
+```
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
+laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
+```
+
+7. Confirm operation of the registry CLI:
+
+```
+laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
+```
+
+## Configure Digital Ocean firewall
+
+Let's open some ports.
+
+1. In the Digital Ocean web console, navigate to your droplet's main page. Select the "Networking" tab and scroll down to "Firewall".
+
+2. Get the port for the running console:
+
+```
+echo http://IP:$(laconic-so --stack fixturenet-laconic-loaded deploy port laconic-console 80 | cut -d ':' -f 2)
+```
+```
+http://IP:32778
+```
+
+3. Go back to the Digital Ocean web console and set an Inbound Rule for Custom TCP of the above port:
+
+- `32778` in this example, but yours will be different.
+- do the same for port `9473`
+
+Additional ports will need to be opened depending on your application. Ensure you add your droplet to this new Firewall and wait a minute or so for the update to propagate.
+
+4. Navigate to http://IP:port and ensure laconic-console is functioning as expected:
+
+- ensure you are connected to `laconicd`; no error message should pop up;
+- the wifi symbol in the bottom right should have a green check mark beside it
+- navigate to the status tab; it should display similar/identical information
+- navigate to the config tab, you'll see something like (with your IP):
+
+```
+wns
+  webui http://68.183.195.210:9473/console
+  server http://68.183.195.210:9473/api
+```
+
+## Publish and query a sample record to the registry
+
+1. The following command will create a bond and publish a record:
+
+```
+laconic-so --stack fixturenet-laconic-loaded deploy exec cli ./scripts/create-demo-records.sh
+```
+
+You'll get an output like:
+
+```
+Balance is: 99998999999999998999600000
+Created bond with id: dd88e8d6f9567b32b28e70552aea4419c5dd3307ebae85a284d1fe38904e301a
+Published demo-record-1.yml with id: bafyreierh3xnfivexlscdwubvczmddsnf46uytyfvrbdhkjzztvsz6ruly
+```
+
+The sample record we deployed looks like:
+
+```
+TODO
+```
+
+2. Return to the laconic-console
+
+- the published record should now be viewable
+- explore it for more information
+- click on the link that opens the GraphQL console
+- the query is pre-loaded, click the button to run it
+- inspect the output
+
+3. Try out additional CLI commands
+
+- these are documented [here](https://github.com/cerc-io/laconic-registry-cli#readme) and updates are forthcoming
+- e.g,:
+
+```
+laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns record list"
+```