Fix lint errors (#523)

* Fix lint errors

* Fix more lint errors

.github/workflows/lint.yml

@@ -0,0 +1,21 @@
+name: Lint Checks
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+
+jobs:
+  test:
+    name: "Run linter"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name : "Run flake8"
+        uses: py-actions/flake8@v2

app/base.py

@@ -63,7 +63,8 @@ class package_registry_stack(base_stack):
                 self.url = "http://gitea.local:3000/api/packages/cerc-io/npm/"
             else:
                 # If not, print a message about how to start it and return fail to the caller
-                print("ERROR: The package-registry stack is not running, and no external registry specified with CERC_NPM_REGISTRY_URL")
+                print("ERROR: The package-registry stack is not running, and no external registry "
+                      "specified with CERC_NPM_REGISTRY_URL")
                 print("ERROR: Start the local package registry with: laconic-so --stack package-registry deploy-system up")
                 return False
         return True
@@ -75,5 +76,7 @@ class package_registry_stack(base_stack):
 def get_npm_registry_url():
     # If an auth token is not defined, we assume the default should be the cerc registry
     # If an auth token is defined, we assume the local gitea should be used.
-    default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config("CERC_NPM_AUTH_TOKEN", default=None) else "https://git.vdb.to/api/packages/cerc-io/npm/"
+    default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config(
+        "CERC_NPM_AUTH_TOKEN", default=None
+        ) else "https://git.vdb.to/api/packages/cerc-io/npm/"
     return config("CERC_NPM_REGISTRY_URL", default=default_npm_registry_url)

app/build_npms.py

@@ -30,10 +30,12 @@ from app.util import include_exclude_check, get_parsed_stack_config
 
 builder_js_image_name = "cerc/builder-js:local"
 
+
 @click.command()
 @click.option('--include', help="only build these packages")
 @click.option('--exclude', help="don\'t build these packages")
-@click.option("--force-rebuild", is_flag=True, default=False, help="Override existing target package version check -- force rebuild")
+@click.option("--force-rebuild", is_flag=True, default=False,
+              help="Override existing target package version check -- force rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
 @click.pass_context
 def command(ctx, include, exclude, force_rebuild, extra_build_args):
@@ -122,7 +124,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
             # envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             # but that isn't available in Python 3.8 (default in Ubuntu 20) so for now we use dict.update:
             envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token,
-                    "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml" # Convention used by our web app packages
+                    "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml"  # Convention used by our web app packages
                     }
             envs.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             envs.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})

app/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -15,7 +15,7 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473:9473"
+      - "9473"
       - "8545"
       - "8546"
       - "9090"

app/data/compose/docker-compose-mainnet-laconicd.yml

@@ -17,7 +17,7 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473:9473"
+      - "9473"
       - "8545"
       - "8546"
       - "9090"

app/data/compose/docker-compose-mobymask-app.yml

@@ -14,7 +14,7 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_RELEASE: "v0.1.5"
+      CERC_RELEASE: "v0.1.6"
       CERC_USE_NPM: true
       CERC_CONFIG_FILE: "src/config.json"
     working_dir: /scripts
@@ -48,7 +48,7 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_RELEASE: "v0.1.5-lxdao-0.1.1"
+      CERC_RELEASE: "v0.1.6-lxdao-0.1.1"
       CERC_USE_NPM: false
       CERC_CONFIG_FILE: "src/utils/config.json"
     working_dir: /scripts

app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py

@@ -101,7 +101,8 @@ if __name__ == "__main__":
             if args.ssh_host:
                 dst_dbport = random.randint(11000, 12000)
                 print(
-                    f"Establishing SSH tunnel from 127.0.0.1:{dst_dbport} to {args.ssh_host}->{args.dst_dbhost}:{args.dst_dbport}... ",
+                    f"Establishing SSH tunnel from 127.0.0.1:{dst_dbport} to "
+                    "{args.ssh_host}->{args.dst_dbhost}:{args.dst_dbport}... ",
                     end="",
                 )
                 with Connection(

app/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py

@@ -1,5 +1,4 @@
 from web3.auto import w3
-import json
 import ruamel.yaml as yaml
 import sys
 

app/data/stacks/build-support/stack.yml

@@ -1,5 +1,5 @@
 version: "1.2"
 name: build-support
-decription: "Build Support Components"
+description: "Build Support Components"
 containers:
   - cerc/builder-js

app/data/stacks/chain-chunker/stack.yml

@@ -1,6 +1,6 @@
 version: "1.0"
 name: chain-chunker
-decription: "Stack to build containers for chain-chunker"
+description: "Stack to build containers for chain-chunker"
 repos:
   - github.com/cerc-io/ipld-eth-state-snapshot@v5
   - github.com/cerc-io/eth-statediff-service@v5

app/data/stacks/fixturenet-eth-loaded/stack.yml

@@ -1,6 +1,6 @@
 version: "1.0"
 name: fixturenet-eth-loaded
-decription: "Loaded Ethereum Fixturenet"
+description: "Loaded Ethereum Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/tx-spammer

app/data/stacks/fixturenet-eth-tx/stack.yml

@@ -1,6 +1,6 @@
 version: "1.2"
 name: fixturenet-eth-tx
-decription: "Ethereum Fixturenet w/ tx-spammer"
+description: "Ethereum Fixturenet w/ tx-spammer"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/tx-spammer

app/data/stacks/fixturenet-eth/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: fixturenet-eth
-decription: "Ethereum Fixturenet"
+description: "Ethereum Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse

app/data/stacks/fixturenet-optimism/stack.yml

@@ -1,6 +1,6 @@
 version: "1.0"
 name: fixturenet-optimism
-decription: "Optimism Fixturenet"
+description: "Optimism Fixturenet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse

app/data/stacks/fixturenet-plugeth-tx/stack.yml

@@ -1,6 +1,6 @@
 version: "1.2"
 name: fixturenet-plugeth-tx
-decription: "plugeth Ethereum Fixturenet w/ tx-spammer"
+description: "plugeth Ethereum Fixturenet w/ tx-spammer"
 repos:
   - git.vdb.to/cerc-io/plugeth@statediff
   - git.vdb.to/cerc-io/plugeth-statediff

app/data/stacks/mainnet-eth/README.md

@@ -73,7 +73,7 @@ After deleting the volumes, any subsequent re-start will begin chain sync from c
 
 ## Ports
 It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
-Any ports defined in the Docker compose file are exposed by default with random port assignments, but the values can be
+Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be
 customized by editing the "spec" file generated by `laconic-so deploy init`.
 
 In this example, ports `8545` and `5052` have been assigned to a specific addresses/port combination on the host, while
@@ -92,7 +92,15 @@ volumes:
   mainnet_eth_geth_1_data: ./data/mainnet_eth_geth_1_data
   mainnet_eth_lighthouse_1_data: ./data/mainnet_eth_lighthouse_1_data
 ```
-
+In addition, a stack-wide port mapping "recipe" can be applied at the time the 
+`laconic-so deploy init` command is run, by supplying the  desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
+| Recipe | Host Port Mapping |
+|--------|-------------------|
+| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
+| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
+| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
+| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
+| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
 ## Data volumes
 Container data volumes are bind-mounted to specified paths in the host filesystem.
 The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory:

app/data/stacks/mainnet-eth/deploy/commands.py

@@ -15,6 +15,7 @@
 
 from secrets import token_hex
 
+
 def init(ctx):
     return None
 

app/data/stacks/mainnet-eth/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: mainnet-eth
-decription: "Ethereum Mainnet"
+description: "Ethereum Mainnet"
 repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse

app/data/stacks/mainnet-go-opera/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: mainnet-opera
-decription: "Fantom mainnet node"
+description: "Fantom mainnet node"
 repos:
   - github.com/Fantom-foundation/go-opera@release/1.1.2-rc.5
 containers:

app/data/stacks/mainnet-laconic/deploy/commands.py

@@ -32,9 +32,6 @@ default_spec_file_content = """config:
     chain_id: my-chain-id
 """
 
-init_help_text = """Add helpful text here on setting config variables.
-"""
-
 
 class SetupPhase(Enum):
     INITIALIZE = 1
@@ -275,7 +272,6 @@ def create(command_context: DeployCommandContext, extra_args):
 
 
 def init(command_context: DeployCommandContext):
-    print(init_help_text)
     yaml = get_yaml()
     return yaml.load(default_spec_file_content)
 

app/data/stacks/mobymask-v2/web-apps.md

@@ -6,6 +6,12 @@ Instructions to setup and deploy MobyMask and Peer Test web apps
 
 Prerequisite: Watcher with GQL and relay node endpoints
 
+Clone required repositories:
+
+```bash
+laconic-so --stack mobymask-v2 setup-repositories --include github.com/cerc-io/mobymask-ui
+```
+
 Build the container images:
 
 ```bash

app/data/stacks/package-registry/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: package-registry
-decription: "Local Package Registry"
+description: "Local Package Registry"
 repos:
   - github.com/cerc-io/hosting
   - gitea.com/gitea/act_runner

app/data/stacks/reth/stack.yml

@@ -1,6 +1,6 @@
 version: "1.1"
 name: reth
-decription: "Reth node"
+description: "Reth node"
 repos:
   - github.com/paradigmxyz/reth
 containers:

app/data/stacks/test/deploy/commands.py

@@ -23,9 +23,6 @@ default_spec_file_content = """config:
     config_variable: test-value
 """
 
-init_help_text = """Add helpful text here on setting config variables.
-"""
-
 
 # Output a known string to a know file in the bind mounted directory ./container-output-dir
 # for test purposes -- test checks that the file was written.
@@ -40,7 +37,6 @@ def setup(command_context: DeployCommandContext, parameters, extra_args):
 
 
 def init(command_context: DeployCommandContext):
-    print(init_help_text)
     yaml = get_yaml()
     return yaml.load(default_spec_file_content)
 

app/deploy.py

@@ -140,8 +140,8 @@ def exec_operation(ctx, extra_args):
             print(f"Running compose exec {service_name} {command_to_exec}")
         try:
             ctx.obj.docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
-        except DockerException as error:
-            print(f"container command returned error exit status")
+        except DockerException:
+            print("container command returned error exit status")
 
 
 def logs_operation(ctx, tail: int, follow: bool, extra_args: str):

app/deploy_types.py

@@ -19,6 +19,7 @@ from pathlib import Path
 from python_on_whales import DockerClient
 from app.command_types import CommandOptions
 
+
 @dataclass
 class ClusterContext:
     options: CommandOptions  # TODO: this should be in its own object not stuffed in here
@@ -64,4 +65,4 @@ class LaconicStackSetupCommand:
 
 @dataclass
 class LaconicStackCreateCommand:
-    network_dir: str
+    network_dir: str

app/deploy_util.py

@@ -19,7 +19,7 @@ from app.deploy_types import DeployCommandContext, VolumeMapping
 from app.util import get_parsed_stack_config, get_yaml, get_compose_file_dir
 
 
-def _container_image_from_service(stack :str, service: str):
+def _container_image_from_service(stack: str, service: str):
     # Parse the compose files looking for the image name of the specified service
     image_name = None
     parsed_stack = get_parsed_stack_config(stack)

app/deployment.py

@@ -17,8 +17,8 @@ import click
 from dataclasses import dataclass
 from pathlib import Path
 import sys
-from app.deploy import up_operation, down_operation, ps_operation, port_operation, exec_operation, logs_operation, create_deploy_context
-from app.util import global_options
+from app.deploy import up_operation, down_operation, ps_operation, port_operation
+from app.deploy import exec_operation, logs_operation, create_deploy_context
 
 
 @dataclass

app/deployment_create.py

@@ -17,6 +17,7 @@ import click
 from importlib import util
 import os
 from pathlib import Path
+import random
 from shutil import copyfile, copytree
 import sys
 from app.util import get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml
@@ -40,7 +41,7 @@ def _get_ports(stack):
             for svc_name, svc in parsed_pod_file["services"].items():
                 if "ports" in svc:
                     # Ports can appear as strings or numbers.  We normalize them as strings.
-                    ports[svc_name] = [ str(x) for x in svc["ports"] ]
+                    ports[svc_name] = [str(x) for x in svc["ports"]]
     return ports
 
 
@@ -166,10 +167,48 @@ def _find_extra_config_dirs(parsed_pod_file, pod):
     return config_dirs
 
 
+def _get_mapped_ports(stack: str, map_recipe: str):
+    port_map_recipes = ["any-variable-random", "localhost-same", "any-same", "localhost-fixed-random", "any-fixed-random"]
+    ports = _get_ports(stack)
+    if ports:
+        # Implement any requested mapping recipe
+        if map_recipe:
+            if map_recipe in port_map_recipes:
+                for service in ports.keys():
+                    ports_array = ports[service]
+                    for x in range(0, len(ports_array)):
+                        orig_port = ports_array[x]
+                        random_port = random.randint(20000, 50000)  # Beware: we're relying on luck to not collide
+                        if map_recipe == "any-variable-random":
+                            # This is the default so take no action
+                            pass
+                        elif map_recipe == "localhost-same":
+                            # Replace instances of "- XX" with "- 127.0.0.1:XX"
+                            ports_array[x] = f"127.0.0.1:{orig_port}:{orig_port}"
+                        elif map_recipe == "any-same":
+                            # Replace instances of "- XX" with "- 0.0.0.0:XX"
+                            ports_array[x] = f"0.0.0.0:{orig_port}:{orig_port}"
+                        elif map_recipe == "localhost-fixed-random":
+                            # Replace instances of "- XX" with "- 127.0.0.1:<rnd>:XX"
+                            ports_array[x] = f"127.0.0.1:{random_port}:{orig_port}"
+                        elif map_recipe == "any-fixed-random":
+                            # Replace instances of "- XX" with "- 0.0.0.0:<rnd>:XX"
+                            ports_array[x] = f"0.0.0.0:{random_port}:{orig_port}"
+                        else:
+                            print("Error: bad map_recipe")
+            else:
+                print(f"Error: --map-ports-to-host must specify one of: {port_map_recipes}")
+                sys.exit(1)
+    return ports
+
+
 @click.command()
 @click.option("--output", required=True, help="Write yaml spec file here")
+@click.option("--map-ports-to-host", required=False,
+              help="Map ports to the host as one of: any-variable-random (default), "
+              "localhost-same, any-same, localhost-fixed-random, any-fixed-random")
 @click.pass_context
-def init(ctx, output):
+def init(ctx, output, map_ports_to_host):
     yaml = get_yaml()
     stack = global_options(ctx).stack
     verbose = global_options(ctx).verbose
@@ -180,9 +219,8 @@ def init(ctx, output):
     if verbose:
         print(f"Creating spec file for stack: {stack}")
 
-    ports = _get_ports(stack)
-    if ports:
-        spec_file_content["ports"] = ports
+    ports = _get_mapped_ports(stack, map_ports_to_host)
+    spec_file_content["ports"] = ports
 
     named_volumes = _get_named_volumes(stack)
     if named_volumes:

app/setup_repositories.py

@@ -76,7 +76,7 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
     try:
         current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).active_branch.name
         is_branch = True
-    except TypeError as error:
+    except TypeError:
         # This means that the current ref is not a branch, so possibly a tag
         # Let's try to get the tag
         current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
@@ -96,7 +96,9 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
     repoName = repo_path.split("/")[-1]
     full_filesystem_repo_path = os.path.join(dev_root_path, repoName)
     is_present = os.path.isdir(full_filesystem_repo_path)
-    (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(full_filesystem_repo_path) if is_present else (None, None)
+    (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(
+        full_filesystem_repo_path
+        ) if is_present else (None, None)
     if not quiet:
         present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
             else 'Needs to be fetched'
@@ -116,7 +118,7 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
                         origin = git_repo.remotes.origin
                         origin.pull(progress=None if quiet else GitProgress())
                     else:
-                        print(f"skipping pull because this repo checked out a tag")
+                        print("skipping pull because this repo checked out a tag")
                 else:
                     print("(git pull skipped)")
     if not is_present:
@@ -143,7 +145,10 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
         branch_to_checkout = repo_branch
 
     if branch_to_checkout:
-        if current_repo_branch_or_tag is None or (current_repo_branch_or_tag and (current_repo_branch_or_tag != branch_to_checkout)):
+        if current_repo_branch_or_tag is None or (
+            current_repo_branch_or_tag and (
+                current_repo_branch_or_tag != branch_to_checkout)
+                ):
             if not quiet:
                 print(f"switching to branch {branch_to_checkout} in repo {repo_path}")
             git_repo = git.Repo(full_filesystem_repo_path)

app/version.py

@@ -16,6 +16,7 @@
 import click
 import importlib.resources
 
+
 @click.command()
 @click.pass_context
 def command(ctx):

cli.py

@@ -14,7 +14,6 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import click
-from dataclasses import dataclass
 
 from app.command_types import CommandOptions
 from app import setup_repositories