huh

Co-authored-by: David Boreham <david@bozemanpas.com>

.github/workflows/lint.yml

@@ -0,0 +1,21 @@
+name: Lint Checks
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+
+jobs:
+  test:
+    name: "Run linter"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name : "Run flake8"
+        uses: py-actions/flake8@v2

app/base.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -63,7 +63,8 @@ class package_registry_stack(base_stack):
                 self.url = "http://gitea.local:3000/api/packages/cerc-io/npm/"
             else:
                 # If not, print a message about how to start it and return fail to the caller
-                print("ERROR: The package-registry stack is not running, and no external registry specified with CERC_NPM_REGISTRY_URL")
+                print("ERROR: The package-registry stack is not running, and no external registry "
+                      "specified with CERC_NPM_REGISTRY_URL")
                 print("ERROR: Start the local package registry with: laconic-so --stack package-registry deploy-system up")
                 return False
         return True
@@ -75,5 +76,7 @@ class package_registry_stack(base_stack):
 def get_npm_registry_url():
     # If an auth token is not defined, we assume the default should be the cerc registry
     # If an auth token is defined, we assume the local gitea should be used.
-    default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config("CERC_NPM_AUTH_TOKEN", default=None) else "https://git.vdb.to/api/packages/cerc-io/npm/"
+    default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config(
+        "CERC_NPM_AUTH_TOKEN", default=None
+        ) else "https://git.vdb.to/api/packages/cerc-io/npm/"
     return config("CERC_NPM_REGISTRY_URL", default=default_npm_registry_url)

app/build_containers.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -86,6 +86,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     # TODO: make this configurable
     container_build_env = {
         "CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
+        "CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""),
         "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""),
         "CERC_REPO_BASE_DIR": dev_root_path,
         "CERC_CONTAINER_BASE_DIR": container_build_dir,

app/build_npms.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -30,10 +30,12 @@ from app.util import include_exclude_check, get_parsed_stack_config
 
 builder_js_image_name = "cerc/builder-js:local"
 
+
 @click.command()
 @click.option('--include', help="only build these packages")
 @click.option('--exclude', help="don\'t build these packages")
-@click.option("--force-rebuild", is_flag=True, default=False, help="Override existing target package version check -- force rebuild")
+@click.option("--force-rebuild", is_flag=True, default=False,
+              help="Override existing target package version check -- force rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
 @click.pass_context
 def command(ctx, include, exclude, force_rebuild, extra_build_args):
@@ -122,7 +124,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
             # envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             # but that isn't available in Python 3.8 (default in Ubuntu 20) so for now we use dict.update:
             envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token,
-                    "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml" # Convention used by our web app packages
+                    "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml"  # Convention used by our web app packages
                     }
             envs.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             envs.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})

app/command_types.py

@@ -1,4 +1,4 @@
-# Copyright © 2023 Cerc
+# Copyright © 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

app/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -15,7 +15,7 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473:9473"
+      - "9473"
       - "8545"
       - "8546"
       - "9090"

app/data/compose/docker-compose-mainnet-laconicd.yml

@@ -17,7 +17,7 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473:9473"
+      - "9473"
       - "8545"
       - "8546"
       - "9090"

app/data/compose/docker-compose-mobymask-app.yml

@@ -14,7 +14,8 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_RELEASE: "v0.1.5"
+      CERC_PUBSUB: ${CERC_PUBSUB}
+      CERC_RELEASE: "v0.1.7"
       CERC_USE_NPM: true
       CERC_CONFIG_FILE: "src/config.json"
     working_dir: /scripts
@@ -48,7 +49,8 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_RELEASE: "v0.1.5-lxdao-0.1.1"
+      CERC_PUBSUB: ${CERC_PUBSUB}
+      CERC_RELEASE: "v0.1.7-lxdao-0.1.1"
       CERC_USE_NPM: false
       CERC_CONFIG_FILE: "src/utils/config.json"
     working_dir: /scripts

app/data/compose/docker-compose-watcher-mobymask-v2.yml

@@ -84,6 +84,7 @@ services:
       CERC_PRIVATE_KEY_PEER: ${CERC_PRIVATE_KEY_PEER}
       CERC_RELAY_PEERS: ${CERC_RELAY_PEERS}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
+      CERC_PUBSUB: ${CERC_PUBSUB}
       CERC_RELAY_ANNOUNCE_DOMAIN: ${CERC_RELAY_ANNOUNCE_DOMAIN}
       CERC_ENABLE_PEER_L2_TXS: ${CERC_ENABLE_PEER_L2_TXS}
       CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}

app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py

@@ -101,7 +101,8 @@ if __name__ == "__main__":
             if args.ssh_host:
                 dst_dbport = random.randint(11000, 12000)
                 print(
-                    f"Establishing SSH tunnel from 127.0.0.1:{dst_dbport} to {args.ssh_host}->{args.dst_dbhost}:{args.dst_dbport}... ",
+                    f"Establishing SSH tunnel from 127.0.0.1:{dst_dbport} to "
+                    "{args.ssh_host}->{args.dst_dbhost}:{args.dst_dbport}... ",
                     end="",
                 )
                 with Connection(

app/data/config/watcher-mobymask-v2/mobymask-app-start.sh

@@ -8,6 +8,7 @@ CERC_CHAIN_ID="${CERC_CHAIN_ID:-${DEFAULT_CERC_CHAIN_ID}}"
 CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRACT}}"
 CERC_RELAY_NODES="${CERC_RELAY_NODES:-${DEFAULT_CERC_RELAY_NODES}}"
 CERC_DENY_MULTIADDRS="${CERC_DENY_MULTIADDRS:-${DEFAULT_CERC_DENY_MULTIADDRS}}"
+CERC_PUBSUB="${CERC_PUBSUB:-${DEFAULT_CERC_PUBSUB}}"
 CERC_APP_WATCHER_URL="${CERC_APP_WATCHER_URL:-${DEFAULT_CERC_APP_WATCHER_URL}}"
 
 # If not set (or []), check the mounted volume for relay peer id
@@ -41,7 +42,8 @@ jq --arg address "$CERC_DEPLOYED_CONTRACT" \
   --argjson chainId "$CERC_CHAIN_ID" \
   --argjson relayNodes "$CERC_RELAY_NODES" \
   --argjson denyMultiaddrs "$CERC_DENY_MULTIADDRS" \
-  '.address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.denyMultiaddrs = $denyMultiaddrs' \
+  --arg pubsub "$CERC_PUBSUB" \
+  '.address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.denyMultiaddrs = $denyMultiaddrs | .peer.pubsub = $pubsub' \
   /app/src/mobymask-app-config.json > /app/${CERC_CONFIG_FILE}
 
 if [ "${CERC_USE_NPM}" = "true" ]; then

app/data/config/watcher-mobymask-v2/mobymask-params.env

@@ -27,3 +27,6 @@ DEFAULT_CERC_RELAY_NODES=[]
 
 # Set of multiaddrs to be avoided while dialling
 DEFAULT_CERC_DENY_MULTIADDRS=[]
+
+# Type of pubsub to be used
+DEFAULT_CERC_PUBSUB=""

app/data/config/watcher-mobymask-v2/start-server.sh

@@ -9,6 +9,7 @@ CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS
 
 CERC_RELAY_PEERS="${CERC_RELAY_PEERS:-${DEFAULT_CERC_RELAY_PEERS}}"
 CERC_DENY_MULTIADDRS="${CERC_DENY_MULTIADDRS:-${DEFAULT_CERC_DENY_MULTIADDRS}}"
+CERC_PUBSUB="${CERC_PUBSUB:-${DEFAULT_CERC_PUBSUB}}"
 CERC_RELAY_ANNOUNCE_DOMAIN="${CERC_RELAY_ANNOUNCE_DOMAIN:-${DEFAULT_CERC_RELAY_ANNOUNCE_DOMAIN}}"
 CERC_ENABLE_PEER_L2_TXS="${CERC_ENABLE_PEER_L2_TXS:-${DEFAULT_CERC_ENABLE_PEER_L2_TXS}}"
 CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRACT}}"
@@ -50,6 +51,7 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
   sed -E "s|REPLACE_WITH_CERC_RELAY_PEERS|${CERC_RELAY_PEERS}|g; \
           s|REPLACE_WITH_CERC_DENY_MULTIADDRS|${CERC_DENY_MULTIADDRS}|g; \
+          s/REPLACE_WITH_CERC_PUBSUB/${CERC_PUBSUB}/g; \
           s/REPLACE_WITH_CERC_RELAY_ANNOUNCE_DOMAIN/${CERC_RELAY_ANNOUNCE_DOMAIN}/g; \
           s|REPLACE_WITH_CERC_RELAY_MULTIADDR|${CERC_RELAY_MULTIADDR}|g; \
           s/REPLACE_WITH_CERC_ENABLE_PEER_L2_TXS/${CERC_ENABLE_PEER_L2_TXS}/g; \

app/data/config/watcher-mobymask-v2/watcher-config-template.toml

@@ -30,6 +30,7 @@
       denyMultiaddrs = REPLACE_WITH_CERC_DENY_MULTIADDRS
       peerIdFile = './peers/relay-id.json'
       announce = 'REPLACE_WITH_CERC_RELAY_ANNOUNCE_DOMAIN'
+      pubsub = 'REPLACE_WITH_CERC_PUBSUB'
       enableDebugInfo = true
 
     [server.p2p.peer]
@@ -37,6 +38,7 @@
       pubSubTopic = 'mobymask'
       denyMultiaddrs = REPLACE_WITH_CERC_DENY_MULTIADDRS
       peerIdFile = './peers/peer-id.json'
+      pubsub = 'REPLACE_WITH_CERC_PUBSUB'
       enableDebugInfo = true
       enableL2Txs = REPLACE_WITH_CERC_ENABLE_PEER_L2_TXS
 

app/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py

@@ -1,5 +1,4 @@
 from web3.auto import w3
-import json
 import ruamel.yaml as yaml
 import sys
 

app/data/container-build/cerc-plugeth-statediff/build.sh

@@ -1,4 +1,10 @@
 #!/usr/bin/env bash
 # Build cerc/plugeth-statediff
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-docker build -t cerc/plugeth-statediff:local ${build_command_args} ${CERC_REPO_BASE_DIR}/plugeth-statediff
+# This container build currently requires access to private dependencies in gitea
+# so we check that the necessary access token has been supplied here, then pass it o the build
+if [[ -z "${CERC_GO_AUTH_TOKEN}" ]]; then
+    echo "ERROR: CERC_GO_AUTH_TOKEN is not set" >&2
+    exit 1
+fi
+docker build -t cerc/plugeth-statediff:local ${build_command_args} --build-arg GIT_VDBTO_TOKEN=${CERC_GO_AUTH_TOKEN} ${CERC_REPO_BASE_DIR}/plugeth-statediff

app/data/container-build/cerc-plugeth/build.sh

@@ -1,4 +1,10 @@
 #!/usr/bin/env bash
 # Build cerc/plugeth
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-docker build -t cerc/plugeth:local ${build_command_args} ${CERC_REPO_BASE_DIR}/plugeth
+# This container build currently requires access to private dependencies in gitea
+# so we check that the necessary access token has been supplied here, then pass it o the build
+if [[ -z "${CERC_GO_AUTH_TOKEN}" ]]; then
+    echo "ERROR: CERC_GO_AUTH_TOKEN is not set" >&2
+    exit 1
+fi
+docker build -t cerc/plugeth:local ${build_command_args} --build-arg GIT_VDBTO_TOKEN=${CERC_GO_AUTH_TOKEN} ${CERC_REPO_BASE_DIR}/plugeth

app/data/container-build/cerc-watcher-erc20/build.sh

@@ -6,4 +6,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/watcher-erc20:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/watcher-ts
+docker build -t cerc/watcher-erc20:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/erc20-watcher-ts

app/data/stacks/build-support/stack.yml

@@ -1,5 +1,5 @@
 version: "1.2"
 name: build-support
-decription: "Build Support Components"
+description: "Build Support Components"
 containers:
   - cerc/builder-js

app/data/stacks/chain-chunker/stack.yml

@@ -1,6 +1,6 @@
 version: "1.0"
 name: chain-chunker
-decription: "Stack to build containers for chain-chunker"
+description: "Stack to build containers for chain-chunker"
 repos:
   - github.com/cerc-io/ipld-eth-state-snapshot@v5
   - github.com/cerc-io/eth-statediff-service@v5

app/data/stacks/erc20/stack.yml

@@ -4,7 +4,7 @@ repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/ipld-eth-db
   - github.com/cerc-io/ipld-eth-server
-  - github.com/cerc-io/watcher-ts
+  - github.com/cerc-io/erc20-watcher-ts
   - github.com/dboreham/foundry
 containers:
   - cerc/foundry

app/data/stacks/fixturenet-eth-loaded/stack.yml

@@ -1,6 +1,6 @@
 version: "1.0"
 name: fixturenet-eth-loaded
-decription: "Loaded Ethereum Fixturenet"
+description: "Loaded Ethereum Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/tx-spammer

app/data/stacks/fixturenet-eth-tx/stack.yml

@@ -1,6 +1,6 @@
 version: "1.2"
 name: fixturenet-eth-tx
-decription: "Ethereum Fixturenet w/ tx-spammer"
+description: "Ethereum Fixturenet w/ tx-spammer"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/tx-spammer

app/data/stacks/fixturenet-eth/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: fixturenet-eth
-decription: "Ethereum Fixturenet"
+description: "Ethereum Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse

app/data/stacks/fixturenet-optimism/README.md

@@ -18,6 +18,11 @@ Build the container images:
 
 ```bash
 laconic-so --stack fixturenet-optimism build-containers
+
+# If redeploying with changes in the stack containers
+laconic-so --stack fixturenet-optimism build-containers --force-rebuild
+
+# If errors are thrown during build, old images used by this stack would have to be deleted
 ```
 
 Note: this will take >10 mins depending on the specs of your machine, and **requires** 16GB of memory or greater.

app/data/stacks/fixturenet-optimism/stack.yml

@@ -1,6 +1,6 @@
 version: "1.0"
 name: fixturenet-optimism
-decription: "Optimism Fixturenet"
+description: "Optimism Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse

app/data/stacks/fixturenet-plugeth-tx/README.md

@@ -12,6 +12,7 @@ See `stacks/fixturenet-eth/README.md` for more information.
 * cerc/tx-spammer
 
 ## Deploy the stack
+Note: since some Go dependencies are currently private, `CERC_GO_AUTH_TOKEN` must be set to a valid Gitea access token before running the `build-containers` command.
 ```
 $ laconic-so --stack fixturenet-plugeth-tx setup-repositories
 $ laconic-so --stack fixturenet-plugeth-tx build-containers

app/data/stacks/fixturenet-plugeth-tx/stack.yml

@@ -1,6 +1,6 @@
 version: "1.2"
 name: fixturenet-plugeth-tx
-decription: "plugeth Ethereum Fixturenet w/ tx-spammer"
+description: "plugeth Ethereum Fixturenet w/ tx-spammer"
 repos:
   - git.vdb.to/cerc-io/plugeth@statediff
   - git.vdb.to/cerc-io/plugeth-statediff

app/data/stacks/mainnet-eth/README.md

@@ -73,7 +73,7 @@ After deleting the volumes, any subsequent re-start will begin chain sync from c
 
 ## Ports
 It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
-Any ports defined in the Docker compose file are exposed by default with random port assignments, but the values can be
+Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be
 customized by editing the "spec" file generated by `laconic-so deploy init`.
 
 In this example, ports `8545` and `5052` have been assigned to a specific addresses/port combination on the host, while
@@ -92,7 +92,15 @@ volumes:
   mainnet_eth_geth_1_data: ./data/mainnet_eth_geth_1_data
   mainnet_eth_lighthouse_1_data: ./data/mainnet_eth_lighthouse_1_data
 ```
-
+In addition, a stack-wide port mapping "recipe" can be applied at the time the 
+`laconic-so deploy init` command is run, by supplying the  desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
+| Recipe | Host Port Mapping |
+|--------|-------------------|
+| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
+| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
+| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
+| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
+| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
 ## Data volumes
 Container data volumes are bind-mounted to specified paths in the host filesystem.
 The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory:

app/data/stacks/mainnet-eth/deploy/commands.py

@@ -1,4 +1,4 @@
-# Copyright © 2023 Cerc
+# Copyright © 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,6 +15,7 @@
 
 from secrets import token_hex
 
+
 def init(ctx):
     return None
 

app/data/stacks/mainnet-eth/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: mainnet-eth
-decription: "Ethereum Mainnet"
+description: "Ethereum Mainnet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse

app/data/stacks/mainnet-go-opera/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: mainnet-opera
-decription: "Fantom mainnet node"
+description: "Fantom mainnet node"
 repos:
   - github.com/Fantom-foundation/go-opera@release/1.1.2-rc.5
 containers:

app/data/stacks/mainnet-laconic/deploy/commands.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from app.util import get_yaml
-from app.deploy_types import DeployCommandContext, LaconicStackSetupCommand
+from app.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
 from app.stack_state import State
 from app.deploy_util import VolumeMapping, run_container_command
 from app.command_types import CommandOptions
@@ -32,9 +32,6 @@ default_spec_file_content = """config:
     chain_id: my-chain-id
 """
 
-init_help_text = """Add helpful text here on setting config variables.
-"""
-
 
 class SetupPhase(Enum):
     INITIALIZE = 1
@@ -113,17 +110,22 @@ def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
         output_file.write(config_file_content)
 
 
-def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args):
+def _insert_persistent_peers(options: CommandOptions, config_dir: Path, new_persistent_peers: str):
+    config_file_path = config_dir.joinpath("config.toml")
+    if not config_file_path.exists():
+        print("Error: config.toml not found")
+        sys.exit(1)
+    with open(config_file_path, "r") as input_file:
+        config_file_content = input_file.read()
+        persistent_peers_pattern = '^persistent_peers = ""'
+        replace_with = f"persistent_peers = \"{new_persistent_peers}\""
+        config_file_content = re.sub(persistent_peers_pattern, replace_with, config_file_content, flags=re.MULTILINE)
+    with open(config_file_path, "w") as output_file:
+        output_file.write(config_file_content)
 
-    options = command_context.cluster_context.options
-
-    currency = "stake"  # Does this need to be a parameter?
-
-    if options.debug:
-        print(f"parameters: {parameters}")
 
+def _phase_from_params(parameters):
     phase = SetupPhase.ILLEGAL
-
     if parameters.initialize_network:
         if parameters.join_network or parameters.create_network:
             print("Can't supply --join-network or --create-network with --initialize-network")
@@ -146,6 +148,19 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             print("Can't supply --initialize-network or --join-network with --create-network")
             sys.exit(1)
         phase = SetupPhase.CREATE
+    return phase
+
+
+def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args):
+
+    options = command_context.cluster_context.options
+
+    currency = "stake"  # Does this need to be a parameter?
+
+    if options.debug:
+        print(f"parameters: {parameters}")
+
+    phase = _phase_from_params(parameters)
 
     network_dir = Path(parameters.network_dir).absolute()
     laconicd_home_path_in_container = "/laconicd-home"
@@ -187,20 +202,22 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             f"laconicd add-genesis-account {parameters.key_name} 12900000000000000000000{currency}\
                 --home {laconicd_home_path_in_container} --keyring-backend test",
             mounts)
-        print(f"Command output: {output2}")
+        if options.debug:
+            print(f"Command output: {output2}")
         output3, status3 = run_container_command(
             command_context,
             "laconicd",
             f"laconicd gentx  {parameters.key_name} 90000000000{currency} --home {laconicd_home_path_in_container}\
                 --chain-id {chain_id} --keyring-backend test",
             mounts)
-        print(f"Command output: {output3}")
+        if options.debug:
+            print(f"Command output: {output3}")
         output4, status4 = run_container_command(
             command_context,
             "laconicd",
             f"laconicd keys show  {parameters.key_name} -a --home {laconicd_home_path_in_container} --keyring-backend test",
             mounts)
-        print(f"Command output: {output4}")
+        print(f"Node validator address: {output4}")
 
     elif phase == SetupPhase.CREATE:
         if not os.path.exists(network_dir):
@@ -219,6 +236,9 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             copyfile(genesis_file_path, os.path.join(network_dir, "config", os.path.basename(genesis_file_path)))
         else:
             # We're generating the genesis file
+            if not parameters.gentx_file_list:
+                print("Error: --gentx-files must be supplied")
+                sys.exit(1)
             # First look in the supplied gentx files for the other nodes' keys
             other_node_keys = _get_node_keys_from_gentx_files(options, parameters.gentx_file_list)
             # Add those keys to our genesis, with balances we determine here (why?)
@@ -226,13 +246,15 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
                 outputk, statusk = run_container_command(
                     command_context, "laconicd", f"laconicd add-genesis-account {other_node_key} 12900000000000000000000{currency}\
                         --home {laconicd_home_path_in_container} --keyring-backend test", mounts)
-            print(f"Command output: {outputk}")
+            if options.debug:
+                print(f"Command output: {outputk}")
             # Copy the gentx json files into our network dir
             _copy_gentx_files(options, network_dir, parameters.gentx_file_list)
             # Now we can run collect-gentxs
             output1, status1 = run_container_command(
                 command_context, "laconicd", f"laconicd collect-gentxs --home {laconicd_home_path_in_container}", mounts)
-            print(f"Command output: {output1}")
+            if options.debug:
+                print(f"Command output: {output1}")
             print(f"Generated genesis file, please copy to other nodes as required: \
                 {os.path.join(network_dir, 'config', 'genesis.json')}")
             # Last thing, collect-gentxs puts a likely bogus set of persistent_peers in config.toml so we remove that now
@@ -240,15 +262,15 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         # In both cases we validate the genesis file now
         output2, status1 = run_container_command(
             command_context, "laconicd", f"laconicd validate-genesis --home {laconicd_home_path_in_container}", mounts)
-        print(f"Command output: {output2}")
+        print(f"validate-genesis result: {output2}")
 
     else:
         print("Illegal parameters supplied")
         sys.exit(1)
 
 
-def create(command_context: DeployCommandContext, extra_args):
-    network_dir = extra_args
+def create(context: DeploymentContext, extra_args):
+    network_dir = extra_args[0]
     if network_dir is None:
         print("Error: --network-dir must be supplied")
         sys.exit(1)
@@ -266,16 +288,19 @@ def create(command_context: DeployCommandContext, extra_args):
         sys.exit(1)
     # Copy the network directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_config_dir = command_context.deployment_dir.joinpath("data", "laconicd-config")
+    deployment_config_dir = context.deployment_dir.joinpath("data", "laconicd-config")
     copytree(config_dir_path, deployment_config_dir, dirs_exist_ok=True)
+    # If supplied, add the initial persistent peers to the config file
+    if extra_args[1]:
+        initial_persistent_peers = extra_args[1]
+        _insert_persistent_peers(context.command_context.cluster_context.options, deployment_config_dir, initial_persistent_peers)
     # Copy the data directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_data_dir = command_context.deployment_dir.joinpath("data", "laconicd-data")
+    deployment_data_dir = context.deployment_dir.joinpath("data", "laconicd-data")
     copytree(data_dir_path, deployment_data_dir, dirs_exist_ok=True)
 
 
 def init(command_context: DeployCommandContext):
-    print(init_help_text)
     yaml = get_yaml()
     return yaml.load(default_spec_file_content)
 

app/data/stacks/mobymask-v2/mobymask-only.md

@@ -60,6 +60,9 @@ Create and update an env file to be used in the next step ([defaults](../../conf
   # (Optional) Set of multiaddrs to be avoided while dialling
   CERC_DENY_MULTIADDRS=[]
 
+  # (Optional) Type of pubsub to be used
+  CERC_PUBSUB=""
+
   # Set to false for disabling watcher peer to send txs to L2
   CERC_ENABLE_PEER_L2_TXS=true
 
@@ -119,4 +122,8 @@ docker volume ls -q --filter "name=mobymask_v2"
 
 # Remove all the listed volumes
 docker volume rm $(docker volume ls -q --filter "name=mobymask_v2")
+
+# WARNING: To avoid changing peer ids for the watcher, `peers_ids` volume can be persisted
+# To delete all volumes except for `peers_ids`
+docker volume rm $(docker volume ls -q --filter "name=mobymask_v2" | grep -v "peers_ids$")
 ```

app/data/stacks/mobymask-v2/stack.yml

@@ -6,8 +6,8 @@ repos:
   - github.com/dboreham/foundry
   - github.com/ethereum-optimism/optimism@v1.0.4
   - github.com/ethereum-optimism/op-geth@v1.101105.2
-  - github.com/cerc-io/watcher-ts@v0.2.55
-  - github.com/cerc-io/mobymask-v2-watcher-ts@v0.1.2
+  - github.com/cerc-io/watcher-ts@v0.2.56
+  - github.com/cerc-io/mobymask-v2-watcher-ts@v0.1.3
   - github.com/cerc-io/MobyMask@v0.1.3
   - github.com/cerc-io/mobymask-ui
 containers:

app/data/stacks/mobymask-v2/web-apps.md

@@ -6,6 +6,12 @@ Instructions to setup and deploy MobyMask and Peer Test web apps
 
 Prerequisite: Watcher with GQL and relay node endpoints
 
+Clone required repositories:
+
+```bash
+laconic-so --stack mobymask-v2 setup-repositories --include github.com/cerc-io/mobymask-ui
+```
+
 Build the container images:
 
 ```bash
@@ -39,6 +45,9 @@ Create and update an env file to be used in the next step ([defaults](../../conf
 
   # L2 Chain ID used by mobymask web-app for L2 txs
   CERC_CHAIN_ID=42069
+
+  # (Optional) Type of pubsub to be used ("floodsub" | "gossipsub")
+  CERC_PUBSUB=""
   ```
 
 * NOTE: If watcher is running on the host machine, use `host.docker.internal` as the hostname to access the host port

app/data/stacks/package-registry/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: package-registry
-decription: "Local Package Registry"
+description: "Local Package Registry"
 repos:
   - github.com/cerc-io/hosting
   - gitea.com/gitea/act_runner

app/data/stacks/reth/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: reth
-decription: "Reth node"
+description: "Reth node"
 repos:
   - github.com/paradigmxyz/reth
 containers:

app/data/stacks/test/deploy/commands.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,9 +23,6 @@ default_spec_file_content = """config:
     config_variable: test-value
 """
 
-init_help_text = """Add helpful text here on setting config variables.
-"""
-
 
 # Output a known string to a know file in the bind mounted directory ./container-output-dir
 # for test purposes -- test checks that the file was written.
@@ -40,7 +37,6 @@ def setup(command_context: DeployCommandContext, parameters, extra_args):
 
 
 def init(command_context: DeployCommandContext):
-    print(init_help_text)
     yaml = get_yaml()
     return yaml.load(default_spec_file_content)
 

app/deploy.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -140,8 +140,8 @@ def exec_operation(ctx, extra_args):
             print(f"Running compose exec {service_name} {command_to_exec}")
         try:
             ctx.obj.docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
-        except DockerException as error:
-            print(f"container command returned error exit status")
+        except DockerException:
+            print("container command returned error exit status")
 
 
 def logs_operation(ctx, tail: int, follow: bool, extra_args: str):

app/deploy_types.py

@@ -1,4 +1,4 @@
-# Copyright © 2023 Cerc
+# Copyright © 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,6 +19,7 @@ from pathlib import Path
 from python_on_whales import DockerClient
 from app.command_types import CommandOptions
 
+
 @dataclass
 class ClusterContext:
     options: CommandOptions  # TODO: this should be in its own object not stuffed in here
@@ -64,4 +65,4 @@ class LaconicStackSetupCommand:
 
 @dataclass
 class LaconicStackCreateCommand:
-    network_dir: str
+    network_dir: str

app/deploy_util.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,7 +19,7 @@ from app.deploy_types import DeployCommandContext, VolumeMapping
 from app.util import get_parsed_stack_config, get_yaml, get_compose_file_dir
 
 
-def _container_image_from_service(stack :str, service: str):
+def _container_image_from_service(stack: str, service: str):
     # Parse the compose files looking for the image name of the specified service
     image_name = None
     parsed_stack = get_parsed_stack_config(stack)

app/deployment.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,8 +17,8 @@ import click
 from dataclasses import dataclass
 from pathlib import Path
 import sys
-from app.deploy import up_operation, down_operation, ps_operation, port_operation, exec_operation, logs_operation, create_deploy_context
-from app.util import global_options
+from app.deploy import up_operation, down_operation, ps_operation, port_operation
+from app.deploy import exec_operation, logs_operation, create_deploy_context
 
 
 @dataclass

app/deployment_create.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,6 +17,7 @@ import click
 from importlib import util
 import os
 from pathlib import Path
+import random
 from shutil import copyfile, copytree
 import sys
 from app.util import get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml
@@ -40,7 +41,7 @@ def _get_ports(stack):
             for svc_name, svc in parsed_pod_file["services"].items():
                 if "ports" in svc:
                     # Ports can appear as strings or numbers.  We normalize them as strings.
-                    ports[svc_name] = [ str(x) for x in svc["ports"] ]
+                    ports[svc_name] = [str(x) for x in svc["ports"]]
     return ports
 
 
@@ -166,10 +167,48 @@ def _find_extra_config_dirs(parsed_pod_file, pod):
     return config_dirs
 
 
+def _get_mapped_ports(stack: str, map_recipe: str):
+    port_map_recipes = ["any-variable-random", "localhost-same", "any-same", "localhost-fixed-random", "any-fixed-random"]
+    ports = _get_ports(stack)
+    if ports:
+        # Implement any requested mapping recipe
+        if map_recipe:
+            if map_recipe in port_map_recipes:
+                for service in ports.keys():
+                    ports_array = ports[service]
+                    for x in range(0, len(ports_array)):
+                        orig_port = ports_array[x]
+                        random_port = random.randint(20000, 50000)  # Beware: we're relying on luck to not collide
+                        if map_recipe == "any-variable-random":
+                            # This is the default so take no action
+                            pass
+                        elif map_recipe == "localhost-same":
+                            # Replace instances of "- XX" with "- 127.0.0.1:XX"
+                            ports_array[x] = f"127.0.0.1:{orig_port}:{orig_port}"
+                        elif map_recipe == "any-same":
+                            # Replace instances of "- XX" with "- 0.0.0.0:XX"
+                            ports_array[x] = f"0.0.0.0:{orig_port}:{orig_port}"
+                        elif map_recipe == "localhost-fixed-random":
+                            # Replace instances of "- XX" with "- 127.0.0.1:<rnd>:XX"
+                            ports_array[x] = f"127.0.0.1:{random_port}:{orig_port}"
+                        elif map_recipe == "any-fixed-random":
+                            # Replace instances of "- XX" with "- 0.0.0.0:<rnd>:XX"
+                            ports_array[x] = f"0.0.0.0:{random_port}:{orig_port}"
+                        else:
+                            print("Error: bad map_recipe")
+            else:
+                print(f"Error: --map-ports-to-host must specify one of: {port_map_recipes}")
+                sys.exit(1)
+    return ports
+
+
 @click.command()
 @click.option("--output", required=True, help="Write yaml spec file here")
+@click.option("--map-ports-to-host", required=False,
+              help="Map ports to the host as one of: any-variable-random (default), "
+              "localhost-same, any-same, localhost-fixed-random, any-fixed-random")
 @click.pass_context
-def init(ctx, output):
+def init(ctx, output, map_ports_to_host):
     yaml = get_yaml()
     stack = global_options(ctx).stack
     verbose = global_options(ctx).verbose
@@ -180,9 +219,8 @@ def init(ctx, output):
     if verbose:
         print(f"Creating spec file for stack: {stack}")
 
-    ports = _get_ports(stack)
-    if ports:
-        spec_file_content["ports"] = ports
+    ports = _get_mapped_ports(stack, map_ports_to_host)
+    spec_file_content["ports"] = ports
 
     named_volumes = _get_named_volumes(stack)
     if named_volumes:
@@ -200,8 +238,9 @@ def init(ctx, output):
 @click.option("--deployment-dir", help="Create deployment files in this directory")
 # TODO: Hack
 @click.option("--network-dir", help="Network configuration supplied in this directory")
+@click.option("--initial-peers", help="Initial set of persistent peers")
 @click.pass_context
-def create(ctx, spec_file, deployment_dir, network_dir):
+def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
     # This function fails with a useful error message if the file doens't exist
     parsed_spec = get_parsed_deployment_spec(spec_file)
     stack_name = parsed_spec['stack']
@@ -249,7 +288,7 @@ def create(ctx, spec_file, deployment_dir, network_dir):
     deployment_command_context = ctx.obj
     deployment_command_context.stack = stack_name
     deployment_context = DeploymentContext(Path(deployment_dir), deployment_command_context)
-    call_stack_deploy_create(deployment_context, network_dir)
+    call_stack_deploy_create(deployment_context, [network_dir, initial_peers])
 
 
 # TODO: this code should be in the stack .py files but

app/setup_repositories.py

@@ -1,4 +1,4 @@
-# Copyright © 2022 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -76,7 +76,7 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
     try:
         current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).active_branch.name
         is_branch = True
-    except TypeError as error:
+    except TypeError:
         # This means that the current ref is not a branch, so possibly a tag
         # Let's try to get the tag
         current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
@@ -96,7 +96,9 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
     repoName = repo_path.split("/")[-1]
     full_filesystem_repo_path = os.path.join(dev_root_path, repoName)
     is_present = os.path.isdir(full_filesystem_repo_path)
-    (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(full_filesystem_repo_path) if is_present else (None, None)
+    (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(
+        full_filesystem_repo_path
+        ) if is_present else (None, None)
     if not quiet:
         present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
             else 'Needs to be fetched'
@@ -116,7 +118,7 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
                         origin = git_repo.remotes.origin
                         origin.pull(progress=None if quiet else GitProgress())
                     else:
-                        print(f"skipping pull because this repo checked out a tag")
+                        print("skipping pull because this repo checked out a tag")
                 else:
                     print("(git pull skipped)")
     if not is_present:
@@ -143,7 +145,10 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
         branch_to_checkout = repo_branch
 
     if branch_to_checkout:
-        if current_repo_branch_or_tag is None or (current_repo_branch_or_tag and (current_repo_branch_or_tag != branch_to_checkout)):
+        if current_repo_branch_or_tag is None or (
+            current_repo_branch_or_tag and (
+                current_repo_branch_or_tag != branch_to_checkout)
+                ):
             if not quiet:
                 print(f"switching to branch {branch_to_checkout} in repo {repo_path}")
             git_repo = git.Repo(full_filesystem_repo_path)

app/stack_state.py

@@ -1,4 +1,4 @@
-# Copyright © 2023 Cerc
+# Copyright © 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

app/util.py

@@ -1,4 +1,4 @@
-# Copyright © 2022, 2023 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

app/version.py

@@ -1,4 +1,4 @@
-# Copyright © 2023 Cerc
+# Copyright © 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -16,6 +16,7 @@
 import click
 import importlib.resources
 
+
 @click.command()
 @click.pass_context
 def command(ctx):

cli.py

@@ -1,4 +1,4 @@
-# Copyright © 2022 Cerc
+# Copyright © 2022, 2023 Vulcanize
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,6 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import click
-from dataclasses import dataclass
 
 from app.command_types import CommandOptions
 from app import setup_repositories