Add flake8 lint check in CI (#522)

* Implement --map-ports-to-host feature

.github/workflows/lint.yml

@@ -0,0 +1,21 @@
+name: Lint Checks
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+
+jobs:
+  test:
+    name: "Run linter"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name : "Run flake8"
+        uses: py-actions/flake8@v2

app/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -15,7 +15,7 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473:9473"
+      - "9473"
       - "8545"
       - "8546"
       - "9090"

app/data/compose/docker-compose-mainnet-laconicd.yml

@@ -17,7 +17,7 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473:9473"
+      - "9473"
       - "8545"
       - "8546"
       - "9090"

app/data/compose/docker-compose-mobymask-app.yml

@@ -14,7 +14,7 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_RELEASE: "v0.1.5"
+      CERC_RELEASE: "v0.1.6"
       CERC_USE_NPM: true
       CERC_CONFIG_FILE: "src/config.json"
     working_dir: /scripts
@@ -48,7 +48,7 @@ services:
       CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
-      CERC_RELEASE: "v0.1.5-lxdao-0.1.1"
+      CERC_RELEASE: "v0.1.6-lxdao-0.1.1"
       CERC_USE_NPM: false
       CERC_CONFIG_FILE: "src/utils/config.json"
     working_dir: /scripts

app/data/stacks/mainnet-eth/README.md

@@ -73,7 +73,7 @@ After deleting the volumes, any subsequent re-start will begin chain sync from c
 
 ## Ports
 It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
-Any ports defined in the Docker compose file are exposed by default with random port assignments, but the values can be
+Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be
 customized by editing the "spec" file generated by `laconic-so deploy init`.
 
 In this example, ports `8545` and `5052` have been assigned to a specific addresses/port combination on the host, while
@@ -92,7 +92,15 @@ volumes:
   mainnet_eth_geth_1_data: ./data/mainnet_eth_geth_1_data
   mainnet_eth_lighthouse_1_data: ./data/mainnet_eth_lighthouse_1_data
 ```
-
+In addition, a stack-wide port mapping "recipe" can be applied at the time the 
+`laconic-so deploy init` command is run, by supplying the  desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
+| Recipe | Host Port Mapping |
+|--------|-------------------|
+| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
+| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
+| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
+| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
+| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
 ## Data volumes
 Container data volumes are bind-mounted to specified paths in the host filesystem.
 The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory:

app/data/stacks/mainnet-laconic/deploy/commands.py

@@ -32,9 +32,6 @@ default_spec_file_content = """config:
     chain_id: my-chain-id
 """
 
-init_help_text = """Add helpful text here on setting config variables.
-"""
-
 
 class SetupPhase(Enum):
     INITIALIZE = 1
@@ -275,7 +272,6 @@ def create(command_context: DeployCommandContext, extra_args):
 
 
 def init(command_context: DeployCommandContext):
-    print(init_help_text)
     yaml = get_yaml()
     return yaml.load(default_spec_file_content)
 

app/data/stacks/mobymask-v2/web-apps.md

@@ -6,6 +6,12 @@ Instructions to setup and deploy MobyMask and Peer Test web apps
 
 Prerequisite: Watcher with GQL and relay node endpoints
 
+Clone required repositories:
+
+```bash
+laconic-so --stack mobymask-v2 setup-repositories --include github.com/cerc-io/mobymask-ui
+```
+
 Build the container images:
 
 ```bash

app/data/stacks/test/deploy/commands.py

@@ -23,9 +23,6 @@ default_spec_file_content = """config:
     config_variable: test-value
 """
 
-init_help_text = """Add helpful text here on setting config variables.
-"""
-
 
 # Output a known string to a know file in the bind mounted directory ./container-output-dir
 # for test purposes -- test checks that the file was written.
@@ -40,7 +37,6 @@ def setup(command_context: DeployCommandContext, parameters, extra_args):
 
 
 def init(command_context: DeployCommandContext):
-    print(init_help_text)
     yaml = get_yaml()
     return yaml.load(default_spec_file_content)
 

app/deployment_create.py

@@ -17,6 +17,7 @@ import click
 from importlib import util
 import os
 from pathlib import Path
+import random
 from shutil import copyfile, copytree
 import sys
 from app.util import get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml
@@ -166,10 +167,48 @@ def _find_extra_config_dirs(parsed_pod_file, pod):
     return config_dirs
 
 
+def _get_mapped_ports(stack: str, map_recipe: str):
+    port_map_recipes = ["any-variable-random", "localhost-same", "any-same", "localhost-fixed-random", "any-fixed-random"]
+    ports = _get_ports(stack)
+    if ports:
+        # Implement any requested mapping recipe
+        if map_recipe:
+            if map_recipe in port_map_recipes:
+                for service in ports.keys():
+                    ports_array = ports[service]
+                    for x in range(0, len(ports_array)):
+                        orig_port = ports_array[x]
+                        random_port = random.randint(20000,50000) # Beware: we're relying on luck to not collide
+                        if map_recipe == "any-variable-random":
+                            # This is the default so take no action
+                            pass
+                        elif map_recipe == "localhost-same":
+                            # Replace instances of "- XX" with "- 127.0.0.1:XX"
+                            ports_array[x] = f"127.0.0.1:{orig_port}:{orig_port}"
+                        elif map_recipe == "any-same":
+                            # Replace instances of "- XX" with "- 0.0.0.0:XX"
+                            ports_array[x] = f"0.0.0.0:{orig_port}:{orig_port}"
+                        elif map_recipe == "localhost-fixed-random":
+                            # Replace instances of "- XX" with "- 127.0.0.1:<rnd>:XX"
+                            ports_array[x] = f"127.0.0.1:{random_port}:{orig_port}"
+                        elif map_recipe == "any-fixed-random":
+                            # Replace instances of "- XX" with "- 0.0.0.0:<rnd>:XX"
+                            ports_array[x] = f"0.0.0.0:{random_port}:{orig_port}"
+                        else:
+                            print("Error: bad map_recipe")
+            else:
+                print(f"Error: --map-ports-to-host must specify one of: {port_map_recipes}")
+                sys.exit(1)
+    return ports
+
+
+
 @click.command()
 @click.option("--output", required=True, help="Write yaml spec file here")
+@click.option("--map-ports-to-host", required=False,
+              help="Map ports to the host as one of: any-variable-random (default), localhost-same, any-same, localhost-fixed-random, any-fixed-random")
 @click.pass_context
-def init(ctx, output):
+def init(ctx, output, map_ports_to_host):
     yaml = get_yaml()
     stack = global_options(ctx).stack
     verbose = global_options(ctx).verbose
@@ -180,9 +219,8 @@ def init(ctx, output):
     if verbose:
         print(f"Creating spec file for stack: {stack}")
 
-    ports = _get_ports(stack)
-    if ports:
-        spec_file_content["ports"] = ports
+    ports = _get_mapped_ports(stack, map_ports_to_host)
+    spec_file_content["ports"] = ports
 
     named_volumes = _get_named_volumes(stack)
     if named_volumes: