Chain chunker stack (#389)

* Fix bug in default container build flow

* Add convenience stack for chain-chunker

Former-commit-id: 3e78c321b0

app/build_containers.py

@@ -115,7 +115,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
             # TODO: make this less of a hack -- should be specified in some metadata somewhere
             # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
             repo_full_path = os.path.join(dev_root_path, repo_dir)
-            repo_dir_or_build_dir = repo_dir if os.path.exists(repo_full_path) else build_dir
+            repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
             build_command = os.path.join(container_build_dir, "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
         if not dry_run:
             if verbose:

app/data/compose/docker-compose-mobymask-app.yml

@@ -13,6 +13,7 @@ services:
       CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
+      CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
       CERC_BUILD_DIR: "@cerc-io/mobymask-ui/build"
     working_dir: /scripts
     command: ["sh", "mobymask-app-start.sh"]
@@ -44,6 +45,7 @@ services:
       CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
+      CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
       CERC_BUILD_DIR: "@cerc-io/mobymask-ui-lxdao/build"
     working_dir: /scripts
     command: ["sh", "mobymask-app-start.sh"]

app/data/compose/docker-compose-peer-test-app.yml

@@ -10,6 +10,7 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
+      CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
     command: ["sh", "test-app-start.sh"]
     volumes:
       - ../config/wait-for-it.sh:/scripts/wait-for-it.sh

app/data/compose/docker-compose-watcher-mobymask-v2.yml

@@ -83,6 +83,7 @@ services:
       CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL}
       CERC_PRIVATE_KEY_PEER: ${CERC_PRIVATE_KEY_PEER}
       CERC_RELAY_PEERS: ${CERC_RELAY_PEERS}
+      CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
       CERC_RELAY_ANNOUNCE_DOMAIN: ${CERC_RELAY_ANNOUNCE_DOMAIN}
       CERC_ENABLE_PEER_L2_TXS: ${CERC_ENABLE_PEER_L2_TXS}
       CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}

app/data/config/watcher-mobymask-v2/mobymask-app-start.sh

@@ -7,6 +7,7 @@ fi
 CERC_CHAIN_ID="${CERC_CHAIN_ID:-${DEFAULT_CERC_CHAIN_ID}}"
 CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRACT}}"
 CERC_RELAY_NODES="${CERC_RELAY_NODES:-${DEFAULT_CERC_RELAY_NODES}}"
+CERC_DENY_MULTIADDRS="${CERC_DENY_MULTIADDRS:-${DEFAULT_CERC_DENY_MULTIADDRS}}"
 CERC_APP_WATCHER_URL="${CERC_APP_WATCHER_URL:-${DEFAULT_CERC_APP_WATCHER_URL}}"
 
 # If not set (or []), check the mounted volume for relay peer id
@@ -37,5 +38,6 @@ yq -n ".address = env(CERC_DEPLOYED_CONTRACT)" > /config/config.yml
 yq ".watcherUrl = env(CERC_APP_WATCHER_URL)" -i /config/config.yml
 yq ".chainId = env(CERC_CHAIN_ID)" -i /config/config.yml
 yq ".relayNodes = strenv(CERC_RELAY_NODES)" -i /config/config.yml
+yq ".denyMultiaddrs = strenv(CERC_DENY_MULTIADDRS)" -i /config/config.yml
 
 /scripts/start-serving-app.sh

app/data/config/watcher-mobymask-v2/mobymask-params.env

@@ -24,3 +24,6 @@ DEFAULT_CERC_CHAIN_ID=42069
 
 # Set of relay nodes to be used by web-apps
 DEFAULT_CERC_RELAY_NODES=[]
+
+# Set of multiaddrs to be avoided while dialling
+DEFAULT_CERC_DENY_MULTIADDRS=[]

app/data/config/watcher-mobymask-v2/start-server.sh

@@ -8,13 +8,20 @@ CERC_L2_GETH_RPC="${CERC_L2_GETH_RPC:-${DEFAULT_CERC_L2_GETH_RPC}}"
 CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"
 
 CERC_RELAY_PEERS="${CERC_RELAY_PEERS:-${DEFAULT_CERC_RELAY_PEERS}}"
+CERC_DENY_MULTIADDRS="${CERC_DENY_MULTIADDRS:-${DEFAULT_CERC_DENY_MULTIADDRS}}"
 CERC_RELAY_ANNOUNCE_DOMAIN="${CERC_RELAY_ANNOUNCE_DOMAIN:-${DEFAULT_CERC_RELAY_ANNOUNCE_DOMAIN}}"
 CERC_ENABLE_PEER_L2_TXS="${CERC_ENABLE_PEER_L2_TXS:-${DEFAULT_CERC_ENABLE_PEER_L2_TXS}}"
 CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRACT}}"
 
 echo "Using L2 RPC endpoint ${CERC_L2_GETH_RPC}"
 
-CERC_RELAY_MULTIADDR="/dns4/mobymask-watcher-server/tcp/9090/ws/p2p/$(jq -r '.id' /app/peers/relay-id.json)"
+# Use public domain for relay multiaddr in peer config if specified
+# Otherwise, use the docker container's host IP
+if [ -n "$CERC_RELAY_ANNOUNCE_DOMAIN" ]; then
+  CERC_RELAY_MULTIADDR="/dns4/${CERC_RELAY_ANNOUNCE_DOMAIN}/tcp/443/wss/p2p/$(jq -r '.id' /app/peers/relay-id.json)"
+else
+  CERC_RELAY_MULTIADDR="/dns4/mobymask-watcher-server/tcp/9090/ws/p2p/$(jq -r '.id' /app/peers/relay-id.json)"
+fi
 
 # Use contract address from environment variable or set from config.json in mounted volume
 if [ -n "$CERC_DEPLOYED_CONTRACT" ]; then
@@ -42,6 +49,7 @@ fi
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
   sed -E "s|REPLACE_WITH_CERC_RELAY_PEERS|${CERC_RELAY_PEERS}|g; \
+          s|REPLACE_WITH_CERC_DENY_MULTIADDRS|${CERC_DENY_MULTIADDRS}|g; \
           s/REPLACE_WITH_CERC_RELAY_ANNOUNCE_DOMAIN/${CERC_RELAY_ANNOUNCE_DOMAIN}/g; \
           s|REPLACE_WITH_CERC_RELAY_MULTIADDR|${CERC_RELAY_MULTIADDR}|g; \
           s/REPLACE_WITH_CERC_ENABLE_PEER_L2_TXS/${CERC_ENABLE_PEER_L2_TXS}/g; \

app/data/config/watcher-mobymask-v2/test-app-config.json

@@ -1,6 +1,7 @@
 {
   "relayNodes": [],
   "peer": {
+    "denyMultiaddrs": [],
     "enableDebugInfo": true
   }
 }

app/data/config/watcher-mobymask-v2/test-app-start.sh

@@ -5,6 +5,7 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 
 CERC_RELAY_NODES="${CERC_RELAY_NODES:-${DEFAULT_CERC_RELAY_NODES}}"
+CERC_DENY_MULTIADDRS="${CERC_DENY_MULTIADDRS:-${DEFAULT_CERC_DENY_MULTIADDRS}}"
 
 # If not set (or []), check the mounted volume for relay peer id
 if [ -z "$CERC_RELAY_NODES" ] || [ "$CERC_RELAY_NODES" = "[]" ]; then
@@ -16,5 +17,6 @@ echo "Using CERC_RELAY_NODES $CERC_RELAY_NODES"
 
 # Use yq to create config.yml with environment variables
 yq -n ".relayNodes = strenv(CERC_RELAY_NODES)" > /config/config.yml
+yq ".denyMultiaddrs = strenv(CERC_DENY_MULTIADDRS)" -i /config/config.yml
 
 /scripts/start-serving-app.sh

app/data/config/watcher-mobymask-v2/watcher-config-template.toml

@@ -27,6 +27,7 @@
       host = "0.0.0.0"
       port = 9090
       relayPeers = REPLACE_WITH_CERC_RELAY_PEERS
+      denyMultiaddrs = REPLACE_WITH_CERC_DENY_MULTIADDRS
       peerIdFile = './peers/relay-id.json'
       announce = 'REPLACE_WITH_CERC_RELAY_ANNOUNCE_DOMAIN'
       enableDebugInfo = true
@@ -34,6 +35,7 @@
     [server.p2p.peer]
       relayMultiaddr = 'REPLACE_WITH_CERC_RELAY_MULTIADDR'
       pubSubTopic = 'mobymask'
+      denyMultiaddrs = REPLACE_WITH_CERC_DENY_MULTIADDRS
       peerIdFile = './peers/peer-id.json'
       enableDebugInfo = true
       enableL2Txs = REPLACE_WITH_CERC_ENABLE_PEER_L2_TXS

app/data/container-build/cerc-mobymask-ui/Dockerfile

@@ -50,9 +50,9 @@ RUN yarn global add http-server
 
 # Globally install both versions of the payload web app package
 # Install old version of MobyMask web app
-RUN yarn global add @cerc-io/mobymask-ui@0.1.3
+RUN yarn global add @cerc-io/mobymask-ui@0.1.4
 # Install the LXDAO version of MobyMask web app
-RUN yarn global add @cerc-io/mobymask-ui-lxdao@npm:@cerc-io/mobymask-ui@0.1.3-lxdao-0.1.1
+RUN yarn global add @cerc-io/mobymask-ui-lxdao@npm:@cerc-io/mobymask-ui@0.1.4-lxdao-0.1.1
 
 # Expose port for http
 EXPOSE 80

app/data/container-build/cerc-mobymask-ui/apply-webapp-config.sh

@@ -33,7 +33,7 @@ do
     echo "Substituting: ${template_string_to_replace} = ${template_value_to_substitute}"
 
     # TODO: Pass keys to be replaced without double quotes
-    if [[ "$template_string_to_replace" =~ ^${config_prefix}_(relayNodes|chainId)$ ]]; then
+    if [[ "$template_string_to_replace" =~ ^${config_prefix}_(relayNodes|chainId|denyMultiaddrs)$ ]]; then
         find ${webapp_files_dir} -type f -exec sed -i 's#"'"${template_string_to_replace}"'"#'"${template_value_to_substitute}"'#g' {} +
     else
         # Note: we do not escape our strings, on the expectation they do not container the '#' char.

app/data/container-build/cerc-react-peer/Dockerfile

@@ -21,7 +21,7 @@ RUN mkdir -p /config
 RUN yarn global add http-server
 
 # Globally install the payload web app package
-RUN yarn global add @cerc-io/test-app@0.2.33
+RUN yarn global add @cerc-io/test-app@0.2.34
 
 # Expose port for http
 EXPOSE 80

app/data/container-build/cerc-react-peer/apply-webapp-config.sh

@@ -33,7 +33,7 @@ do
     echo "Substituting: ${template_string_to_replace} = ${template_value_to_substitute}"
 
     # TODO: Pass keys to be replaced without double quotes
-    if [[ "$template_string_to_replace" == "${config_prefix}_relayNodes" ]]; then
+    if [[ "$template_string_to_replace" =~ ^${config_prefix}_(relayNodes|denyMultiaddrs)$ ]]; then
         find ${webapp_files_dir} -type f -exec sed -i 's#"'"${template_string_to_replace}"'"#'"${template_value_to_substitute}"'#g' {} +
     else
         # Note: we do not escape our strings, on the expectation they do not container the '#' char.

app/data/container-image-list.txt

@@ -38,3 +38,4 @@ cerc/optimism-op-node
 cerc/optimism-op-proposer
 cerc/pocket
 cerc/watcher-azimuth
+cerc/ipld-eth-state-snapshot

app/data/repository-list.txt

@@ -30,3 +30,4 @@ ethereum-optimism/optimism
 pokt-network/pocket-core
 pokt-network/pocket-core-deployments
 cerc-io/azimuth-watcher-ts
+cerc-io/ipld-eth-state-snapshot

app/data/stacks/build-support/README.md

@@ -30,7 +30,7 @@ $ laconic-so --stack package-registry deploy up
  ⠿ Container laconic-aecc4a21d3a502b14522db97d427e850-server-1  Started                                                                                    1.9s
 New user 'gitea_admin' has been successfully created!
 This is your gitea access token: 84fe66a73698bf11edbdccd0a338236b7d1d5c45. Keep it safe and secure, it can not be fetched again from gitea.
-To use with laconic-so set this environment variable: export CERC_NPM_AUTH_TOKEN=3e493e77b3e83fe9e882f7e3a79dd4d5441c308b
+To use with laconic-so set this environment variable: export CERC_NPM_AUTH_TOKEN=84fe66a73698bf11edbdccd0a338236b7d1d5c45
 Created the organization cerc-io
 Gitea was configured to use host name: gitea.local, ensure that this resolves to localhost, e.g. with sudo vi /etc/hosts
 Success, gitea is properly initialized

app/data/stacks/chain-chunker/stack.yml

@@ -0,0 +1,13 @@
+version: "1.0"
+name: chain-chunker
+decription: "Stack to build containers for chain-chunker"
+repos:
+  - cerc-io/ipld-eth-state-snapshot
+  - cerc-io/eth-statediff-service
+  - cerc-io/ipld-eth-db
+  - cerc-io/ipld-eth-server
+containers:
+  - cerc/ipld-eth-state-snapshot
+  - cerc/eth-statediff-service
+  - cerc/ipld-eth-db
+  - cerc/ipld-eth-server

app/data/stacks/mobymask-v2/README.md

@@ -23,11 +23,11 @@ Checkout to the required versions and branches in repos
 ```bash
 # watcher-ts
 cd ~/cerc/watcher-ts
-git checkout v0.2.39
+git checkout v0.2.41
 
 # mobymask-v2-watcher-ts
 cd ~/cerc/mobymask-v2-watcher-ts
-git checkout v0.1.0
+git checkout v0.1.1
 
 # MobyMask
 cd ~/cerc/MobyMask

app/data/stacks/mobymask-v2/mobymask-only.md

@@ -19,11 +19,11 @@ Checkout to the required versions and branches in repos:
 ```bash
 # watcher-ts
 cd ~/cerc/watcher-ts
-git checkout v0.2.39
+git checkout v0.2.41
 
 # mobymask-v2-watcher-ts
 cd ~/cerc/mobymask-v2-watcher-ts
-git checkout v0.1.0
+git checkout v0.1.1
 
 # MobyMask
 cd ~/cerc/MobyMask
@@ -67,11 +67,14 @@ Create and update an env file to be used in the next step ([defaults](../../conf
   # (used for generating a root invite link after deploying the contract)
   CERC_MOBYMASK_APP_BASE_URI="http://127.0.0.1:3002/#"
 
+  # (Optional) Domain to be used in the relay node's announce address
+  CERC_RELAY_ANNOUNCE_DOMAIN=
+
   # (Optional) Set of relay peers to connect to from the relay node
   CERC_RELAY_PEERS=[]
 
-  # (Optional) Domain to be used in the relay node's announce address
-  CERC_RELAY_ANNOUNCE_DOMAIN=
+  # (Optional) Set of multiaddrs to be avoided while dialling
+  CERC_DENY_MULTIADDRS=[]
 
   # Set to false for disabling watcher peer to send txs to L2
   CERC_ENABLE_PEER_L2_TXS=true

app/data/stacks/mobymask-v2/watcher-p2p-network/watcher.md

@@ -35,11 +35,11 @@ Checkout to the required versions and branches in repos:
   ```bash
   # watcher-ts
   cd ~/cerc/watcher-ts
-  git checkout v0.2.39
+  git checkout v0.2.41
 
   # mobymask-v2-watcher-ts
   cd ~/cerc/mobymask-v2-watcher-ts
-  git checkout v0.1.0
+  git checkout v0.1.1
 
   # MobyMask
   cd ~/cerc/MobyMask

app/data/stacks/mobymask-v2/web-apps.md

@@ -26,6 +26,9 @@ Create and update an env file to be used in the next step ([defaults](../../conf
   # Eg. CERC_RELAY_NODES=["/dns4/example.com/tcp/443/wss/p2p/12D3KooWGHmDDCc93XUWL16FMcTPCGu2zFaMkf67k8HZ4gdQbRDr"]
   CERC_RELAY_NODES=[]
 
+  # Set of multiaddrs to be avoided while dialling
+  CERC_DENY_MULTIADDRS=[]
+
   # Also add if running MobyMask app:
 
   # Watcher endpoint used by the app for GQL queries

docs/CONTRIBUTING.md

@@ -26,7 +26,7 @@ In addition to the pre-requisites listed in the [README](/README.md), the follow
 
 1. Clone this repository:
    ```
-   $ git clone (https://github.com/cerc-io/stack-orchestrator.git
+   $ git clone https://github.com/cerc-io/stack-orchestrator.git
    ```
 
 2. Enter the project directory:
@@ -87,20 +87,27 @@ Use shiv to build a single file Python executable zip archive of laconic-so:
    ```
    $ cp stack-orchetrator/laconic-so ~/bin
    $ laconic-so
-      Usage: python -m laconic-so [OPTIONS] COMMAND [ARGS]...
+      Usage: laconic-so [OPTIONS] COMMAND [ARGS]...
 
       Laconic Stack Orchestrator
 
    Options:
+      --stack TEXT         specify a stack to build/deploy
       --quiet
       --verbose
       --dry-run
-      -h, --help  Show this message and exit.
+      --local-stack
+      --debug
+      --continue-on-error
+      -h, --help           Show this message and exit.
 
    Commands:
       build-containers    build the set of containers required for a complete...
+      build-npms          build the set of npm packages required for a...
+      deploy              deploy a stack
       deploy-system       deploy a stack
       setup-repositories  git clone the set of repositories required to build...
+      version             print tool version
    ```
 
 For cutting releases, use the [shiv build script](/scripts/build_shiv_package.sh).

docs/laconicd-fixturenet.md

@@ -162,7 +162,13 @@ Published demo-record-1.yml with id: bafyreierh3xnfivexlscdwubvczmddsnf46uytyfvr
 The sample record we deployed looks like:
 
 ```
-TODO
+record:
+  type: WebsiteRegistrationRecord
+  url: 'https://cerc.io'
+  repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D
+  build_artifact_cid: QmP8jTG1m9GSDJLCbeWhVSVgEzCPPwXRdCRuJtQ5Tz9Kc9
+  tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
+  version: 1.0.23
 ```
 
 2. Return to the laconic-console