Host port mapping recipes (#521)

* Implement --map-ports-to-host feature

app/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -15,7 +15,7 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473:9473"
+      - "9473"
       - "8545"
       - "8546"
       - "9090"

app/data/compose/docker-compose-graph-node.yml

@@ -20,7 +20,7 @@ services:
       postgres_db: graph-node
       ethereum: ${NETWORK:-filecoin}:${ETH_RPC_ENDPOINT:-https://archive.lotus.vdb.to/rpc/v1}
       GRAPH_LOG: debug
-      ETHEREUM_REORG_THRESHOLD: 16
+      ETHEREUM_REORG_THRESHOLD: 3
     ports:
       - "8000"
       - "8001"

app/data/compose/docker-compose-mainnet-laconicd.yml

@@ -17,7 +17,7 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473:9473"
+      - "9473"
       - "8545"
       - "8546"
       - "9090"

app/data/compose/docker-compose-mobymask-app.yml

@@ -14,7 +14,7 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_RELEASE: "v0.1.5"
+      CERC_RELEASE: "v0.1.6"
       CERC_USE_NPM: true
       CERC_CONFIG_FILE: "src/config.json"
     working_dir: /scripts
@@ -48,7 +48,7 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_RELEASE: "v0.1.5-lxdao-0.1.1"
+      CERC_RELEASE: "v0.1.6-lxdao-0.1.1"
       CERC_USE_NPM: false
       CERC_CONFIG_FILE: "src/utils/config.json"
     working_dir: /scripts

app/data/stacks/build-support/stack.yml

@@ -1,5 +1,5 @@
 version: "1.2"
 name: build-support
-decription: "Build Support Components"
+description: "Build Support Components"
 containers:
   - cerc/builder-js

app/data/stacks/chain-chunker/stack.yml

@@ -1,6 +1,6 @@
 version: "1.0"
 name: chain-chunker
-decription: "Stack to build containers for chain-chunker"
+description: "Stack to build containers for chain-chunker"
 repos:
   - github.com/cerc-io/ipld-eth-state-snapshot@v5
   - github.com/cerc-io/eth-statediff-service@v5

app/data/stacks/fixturenet-eth-loaded/stack.yml

@@ -1,6 +1,6 @@
 version: "1.0"
 name: fixturenet-eth-loaded
-decription: "Loaded Ethereum Fixturenet"
+description: "Loaded Ethereum Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/tx-spammer

app/data/stacks/fixturenet-eth-tx/stack.yml

@@ -1,6 +1,6 @@
 version: "1.2"
 name: fixturenet-eth-tx
-decription: "Ethereum Fixturenet w/ tx-spammer"
+description: "Ethereum Fixturenet w/ tx-spammer"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/tx-spammer

app/data/stacks/fixturenet-eth/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: fixturenet-eth
-decription: "Ethereum Fixturenet"
+description: "Ethereum Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse

app/data/stacks/fixturenet-optimism/stack.yml

@@ -1,6 +1,6 @@
 version: "1.0"
 name: fixturenet-optimism
-decription: "Optimism Fixturenet"
+description: "Optimism Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse

app/data/stacks/fixturenet-plugeth-tx/stack.yml

@@ -1,6 +1,6 @@
 version: "1.2"
 name: fixturenet-plugeth-tx
-decription: "plugeth Ethereum Fixturenet w/ tx-spammer"
+description: "plugeth Ethereum Fixturenet w/ tx-spammer"
 repos:
   - git.vdb.to/cerc-io/plugeth@statediff
   - git.vdb.to/cerc-io/plugeth-statediff

app/data/stacks/mainnet-eth/README.md

@@ -73,7 +73,7 @@ After deleting the volumes, any subsequent re-start will begin chain sync from c
 
 ## Ports
 It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
-Any ports defined in the Docker compose file are exposed by default with random port assignments, but the values can be
+Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be
 customized by editing the "spec" file generated by `laconic-so deploy init`.
 
 In this example, ports `8545` and `5052` have been assigned to a specific addresses/port combination on the host, while
@@ -92,7 +92,15 @@ volumes:
   mainnet_eth_geth_1_data: ./data/mainnet_eth_geth_1_data
   mainnet_eth_lighthouse_1_data: ./data/mainnet_eth_lighthouse_1_data
 ```
-
+In addition, a stack-wide port mapping "recipe" can be applied at the time the 
+`laconic-so deploy init` command is run, by supplying the  desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
+| Recipe | Host Port Mapping |
+|--------|-------------------|
+| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
+| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
+| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
+| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
+| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
 ## Data volumes
 Container data volumes are bind-mounted to specified paths in the host filesystem.
 The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory:

app/data/stacks/mainnet-eth/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: mainnet-eth
-decription: "Ethereum Mainnet"
+description: "Ethereum Mainnet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse

app/data/stacks/mainnet-go-opera/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: mainnet-opera
-decription: "Fantom mainnet node"
+description: "Fantom mainnet node"
 repos:
   - github.com/Fantom-foundation/go-opera@release/1.1.2-rc.5
 containers:

app/data/stacks/mainnet-laconic/deploy/commands.py

@@ -32,9 +32,6 @@ default_spec_file_content = """config:
     chain_id: my-chain-id
 """
 
-init_help_text = """Add helpful text here on setting config variables.
-"""
-
 
 class SetupPhase(Enum):
     INITIALIZE = 1
@@ -275,7 +272,6 @@ def create(command_context: DeployCommandContext, extra_args):
 
 
 def init(command_context: DeployCommandContext):
-    print(init_help_text)
     yaml = get_yaml()
     return yaml.load(default_spec_file_content)
 

app/data/stacks/mobymask-v2/web-apps.md

@@ -6,6 +6,12 @@ Instructions to setup and deploy MobyMask and Peer Test web apps
 
 Prerequisite: Watcher with GQL and relay node endpoints
 
+Clone required repositories:
+
+```bash
+laconic-so --stack mobymask-v2 setup-repositories --include github.com/cerc-io/mobymask-ui
+```
+
 Build the container images:
 
 ```bash

app/data/stacks/package-registry/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: package-registry
-decription: "Local Package Registry"
+description: "Local Package Registry"
 repos:
   - github.com/cerc-io/hosting
   - gitea.com/gitea/act_runner

app/data/stacks/reth/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: reth
-decription: "Reth node"
+description: "Reth node"
 repos:
   - github.com/paradigmxyz/reth
 containers:

app/data/stacks/sushiswap-subgraph/README.md

@@ -113,6 +113,48 @@ http://127.0.0.1:8000/subgraphs/name/sushiswap/blocks/graphql
 http://127.0.0.1:8000/subgraphs/name/sushiswap/v3-filecoin/graphql
 ```
 
+## Set environment variables
+
+* The graph-node environment variable `ETHEREUM_REORG_THRESHOLD` can be set in the deployment compose file
+  ```bash
+  $ cat sushiswap-subgraph-deployment/compose/docker-compose-graph-node.yml
+  services:
+    graph-node:
+      image: cerc/graph-node:local
+      ...
+      environment:
+        ...
+        GRAPH_LOG: debug
+        ETHEREUM_REORG_THRESHOLD: 16
+  ```
+  Change `ETHEREUM_REORG_THRESHOLD` to desired value
+
+  * To restart graph-node with updated values, we need to restart only graph-node compose services
+    * Comment `sushiswap-subgraph-v3` pod in stack.yml so that subgraphs are not deployed again
+      ```bash
+      $ cat sushiswap-subgraph-deployment/stack.yml
+      version: "1.0"
+      name: sushiswap-subgraph
+      ...
+      pods:
+        - graph-node
+        # - sushiswap-subgraph-v3
+      ```
+    * Stop the stack first
+      ```bash
+      laconic-so deployment --dir sushiswap-subgraph-deployment stop
+      ```
+    * Start the stack again (will not start `sushiswap-subgraph-v3` pod)
+      ```
+      laconic-so deployment --dir sushiswap-subgraph-deployment start
+      ```
+  * To check if environment variable has been updated in graph-node container
+      ```bash
+      $ laconic-so deployment --dir sushiswap-subgraph-deployment exec graph-node bash
+      root@dc4d3abe1615:/# echo $ETHEREUM_REORG_THRESHOLD
+      16
+      ```
+
 ## Clean up
 
 Stop all the services running in background run:

app/data/stacks/test/deploy/commands.py

@@ -23,9 +23,6 @@ default_spec_file_content = """config:
     config_variable: test-value
 """
 
-init_help_text = """Add helpful text here on setting config variables.
-"""
-
 
 # Output a known string to a know file in the bind mounted directory ./container-output-dir
 # for test purposes -- test checks that the file was written.
@@ -40,7 +37,6 @@ def setup(command_context: DeployCommandContext, parameters, extra_args):
 
 
 def init(command_context: DeployCommandContext):
-    print(init_help_text)
     yaml = get_yaml()
     return yaml.load(default_spec_file_content)
 

app/deployment_create.py

@@ -17,6 +17,7 @@ import click
 from importlib import util
 import os
 from pathlib import Path
+import random
 from shutil import copyfile, copytree
 import sys
 from app.util import get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml
@@ -166,10 +167,48 @@ def _find_extra_config_dirs(parsed_pod_file, pod):
     return config_dirs
 
 
+def _get_mapped_ports(stack: str, map_recipe: str):
+    port_map_recipes = ["any-variable-random", "localhost-same", "any-same", "localhost-fixed-random", "any-fixed-random"]
+    ports = _get_ports(stack)
+    if ports:
+        # Implement any requested mapping recipe
+        if map_recipe:
+            if map_recipe in port_map_recipes:
+                for service in ports.keys():
+                    ports_array = ports[service]
+                    for x in range(0, len(ports_array)):
+                        orig_port = ports_array[x]
+                        random_port = random.randint(20000,50000) # Beware: we're relying on luck to not collide
+                        if map_recipe == "any-variable-random":
+                            # This is the default so take no action
+                            pass
+                        elif map_recipe == "localhost-same":
+                            # Replace instances of "- XX" with "- 127.0.0.1:XX"
+                            ports_array[x] = f"127.0.0.1:{orig_port}:{orig_port}"
+                        elif map_recipe == "any-same":
+                            # Replace instances of "- XX" with "- 0.0.0.0:XX"
+                            ports_array[x] = f"0.0.0.0:{orig_port}:{orig_port}"
+                        elif map_recipe == "localhost-fixed-random":
+                            # Replace instances of "- XX" with "- 127.0.0.1:<rnd>:XX"
+                            ports_array[x] = f"127.0.0.1:{random_port}:{orig_port}"
+                        elif map_recipe == "any-fixed-random":
+                            # Replace instances of "- XX" with "- 0.0.0.0:<rnd>:XX"
+                            ports_array[x] = f"0.0.0.0:{random_port}:{orig_port}"
+                        else:
+                            print("Error: bad map_recipe")
+            else:
+                print(f"Error: --map-ports-to-host must specify one of: {port_map_recipes}")
+                sys.exit(1)
+    return ports
+
+
+
 @click.command()
 @click.option("--output", required=True, help="Write yaml spec file here")
+@click.option("--map-ports-to-host", required=False,
+              help="Map ports to the host as one of: any-variable-random (default), localhost-same, any-same, localhost-fixed-random, any-fixed-random")
 @click.pass_context
-def init(ctx, output):
+def init(ctx, output, map_ports_to_host):
     yaml = get_yaml()
     stack = global_options(ctx).stack
     verbose = global_options(ctx).verbose
@@ -180,9 +219,8 @@ def init(ctx, output):
     if verbose:
         print(f"Creating spec file for stack: {stack}")
 
-    ports = _get_ports(stack)
-    if ports:
-        spec_file_content["ports"] = ports
+    ports = _get_mapped_ports(stack, map_ports_to_host)
+    spec_file_content["ports"] = ports
 
     named_volumes = _get_named_volumes(stack)
     if named_volumes: