Fix lint errors (#525)

* Configure initial peers through deploy create command

* Re-write config file

app/base.py

@@ -63,7 +63,8 @@ class package_registry_stack(base_stack):
                 self.url = "http://gitea.local:3000/api/packages/cerc-io/npm/"
             else:
                 # If not, print a message about how to start it and return fail to the caller
-                print("ERROR: The package-registry stack is not running, and no external registry specified with CERC_NPM_REGISTRY_URL")
+                print("ERROR: The package-registry stack is not running, and no external registry "
+                      "specified with CERC_NPM_REGISTRY_URL")
                 print("ERROR: Start the local package registry with: laconic-so --stack package-registry deploy-system up")
                 return False
         return True
@@ -75,5 +76,7 @@ class package_registry_stack(base_stack):
 def get_npm_registry_url():
     # If an auth token is not defined, we assume the default should be the cerc registry
     # If an auth token is defined, we assume the local gitea should be used.
-    default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config("CERC_NPM_AUTH_TOKEN", default=None) else "https://git.vdb.to/api/packages/cerc-io/npm/"
+    default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config(
+        "CERC_NPM_AUTH_TOKEN", default=None
+        ) else "https://git.vdb.to/api/packages/cerc-io/npm/"
     return config("CERC_NPM_REGISTRY_URL", default=default_npm_registry_url)

app/build_npms.py

@@ -30,10 +30,12 @@ from app.util import include_exclude_check, get_parsed_stack_config
 
 builder_js_image_name = "cerc/builder-js:local"
 
+
 @click.command()
 @click.option('--include', help="only build these packages")
 @click.option('--exclude', help="don\'t build these packages")
-@click.option("--force-rebuild", is_flag=True, default=False, help="Override existing target package version check -- force rebuild")
+@click.option("--force-rebuild", is_flag=True, default=False,
+              help="Override existing target package version check -- force rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
 @click.pass_context
 def command(ctx, include, exclude, force_rebuild, extra_build_args):
@@ -122,7 +124,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
             # envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             # but that isn't available in Python 3.8 (default in Ubuntu 20) so for now we use dict.update:
             envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token,
-                    "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml" # Convention used by our web app packages
+                    "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml"  # Convention used by our web app packages
                     }
             envs.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             envs.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})

app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py

@@ -101,7 +101,8 @@ if __name__ == "__main__":
             if args.ssh_host:
                 dst_dbport = random.randint(11000, 12000)
                 print(
-                    f"Establishing SSH tunnel from 127.0.0.1:{dst_dbport} to {args.ssh_host}->{args.dst_dbhost}:{args.dst_dbport}... ",
+                    f"Establishing SSH tunnel from 127.0.0.1:{dst_dbport} to "
+                    "{args.ssh_host}->{args.dst_dbhost}:{args.dst_dbport}... ",
                     end="",
                 )
                 with Connection(

app/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py

@@ -1,5 +1,4 @@
 from web3.auto import w3
-import json
 import ruamel.yaml as yaml
 import sys
 

app/data/stacks/mainnet-eth/deploy/commands.py

@@ -15,6 +15,7 @@
 
 from secrets import token_hex
 
+
 def init(ctx):
     return None
 

app/data/stacks/mainnet-laconic/deploy/commands.py

@@ -14,7 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from app.util import get_yaml
-from app.deploy_types import DeployCommandContext, LaconicStackSetupCommand
+from app.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
 from app.stack_state import State
 from app.deploy_util import VolumeMapping, run_container_command
 from app.command_types import CommandOptions
@@ -110,17 +110,22 @@ def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
         output_file.write(config_file_content)
 
 
-def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args):
+def _insert_persistent_peers(options: CommandOptions, config_dir: Path, new_persistent_peers: str):
+    config_file_path = config_dir.joinpath("config.toml")
+    if not config_file_path.exists():
+        print("Error: config.toml not found")
+        sys.exit(1)
+    with open(config_file_path, "r") as input_file:
+        config_file_content = input_file.read()
+        persistent_peers_pattern = '^persistent_peers = ""'
+        replace_with = f"persistent_peers = \"{new_persistent_peers}\""
+        config_file_content = re.sub(persistent_peers_pattern, replace_with, config_file_content, flags=re.MULTILINE)
+    with open(config_file_path, "w") as output_file:
+        output_file.write(config_file_content)
 
-    options = command_context.cluster_context.options
-
-    currency = "stake"  # Does this need to be a parameter?
-
-    if options.debug:
-        print(f"parameters: {parameters}")
 
+def _phase_from_params(parameters):
     phase = SetupPhase.ILLEGAL
-
     if parameters.initialize_network:
         if parameters.join_network or parameters.create_network:
             print("Can't supply --join-network or --create-network with --initialize-network")
@@ -143,6 +148,19 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             print("Can't supply --initialize-network or --join-network with --create-network")
             sys.exit(1)
         phase = SetupPhase.CREATE
+    return phase
+
+
+def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args):
+
+    options = command_context.cluster_context.options
+
+    currency = "stake"  # Does this need to be a parameter?
+
+    if options.debug:
+        print(f"parameters: {parameters}")
+
+    phase = _phase_from_params()
 
     network_dir = Path(parameters.network_dir).absolute()
     laconicd_home_path_in_container = "/laconicd-home"
@@ -184,20 +202,22 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             f"laconicd add-genesis-account {parameters.key_name} 12900000000000000000000{currency}\
                 --home {laconicd_home_path_in_container} --keyring-backend test",
             mounts)
-        print(f"Command output: {output2}")
+        if options.debug:
+            print(f"Command output: {output2}")
         output3, status3 = run_container_command(
             command_context,
             "laconicd",
             f"laconicd gentx  {parameters.key_name} 90000000000{currency} --home {laconicd_home_path_in_container}\
                 --chain-id {chain_id} --keyring-backend test",
             mounts)
-        print(f"Command output: {output3}")
+        if options.debug:
+            print(f"Command output: {output3}")
         output4, status4 = run_container_command(
             command_context,
             "laconicd",
             f"laconicd keys show  {parameters.key_name} -a --home {laconicd_home_path_in_container} --keyring-backend test",
             mounts)
-        print(f"Command output: {output4}")
+        print(f"Node validator address: {output4}")
 
     elif phase == SetupPhase.CREATE:
         if not os.path.exists(network_dir):
@@ -216,6 +236,9 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             copyfile(genesis_file_path, os.path.join(network_dir, "config", os.path.basename(genesis_file_path)))
         else:
             # We're generating the genesis file
+            if not parameters.gentx_file_list:
+                print("Error: --gentx-files must be supplied")
+                sys.exit(1)
             # First look in the supplied gentx files for the other nodes' keys
             other_node_keys = _get_node_keys_from_gentx_files(options, parameters.gentx_file_list)
             # Add those keys to our genesis, with balances we determine here (why?)
@@ -223,13 +246,15 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
                 outputk, statusk = run_container_command(
                     command_context, "laconicd", f"laconicd add-genesis-account {other_node_key} 12900000000000000000000{currency}\
                         --home {laconicd_home_path_in_container} --keyring-backend test", mounts)
-            print(f"Command output: {outputk}")
+            if options.debug:
+                print(f"Command output: {outputk}")
             # Copy the gentx json files into our network dir
             _copy_gentx_files(options, network_dir, parameters.gentx_file_list)
             # Now we can run collect-gentxs
             output1, status1 = run_container_command(
                 command_context, "laconicd", f"laconicd collect-gentxs --home {laconicd_home_path_in_container}", mounts)
-            print(f"Command output: {output1}")
+            if options.debug:
+                print(f"Command output: {output1}")
             print(f"Generated genesis file, please copy to other nodes as required: \
                 {os.path.join(network_dir, 'config', 'genesis.json')}")
             # Last thing, collect-gentxs puts a likely bogus set of persistent_peers in config.toml so we remove that now
@@ -237,15 +262,15 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         # In both cases we validate the genesis file now
         output2, status1 = run_container_command(
             command_context, "laconicd", f"laconicd validate-genesis --home {laconicd_home_path_in_container}", mounts)
-        print(f"Command output: {output2}")
+        print(f"validate-genesis result: {output2}")
 
     else:
         print("Illegal parameters supplied")
         sys.exit(1)
 
 
-def create(command_context: DeployCommandContext, extra_args):
-    network_dir = extra_args
+def create(context: DeploymentContext, extra_args):
+    network_dir = extra_args[0]
     if network_dir is None:
         print("Error: --network-dir must be supplied")
         sys.exit(1)
@@ -263,11 +288,15 @@ def create(command_context: DeployCommandContext, extra_args):
         sys.exit(1)
     # Copy the network directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_config_dir = command_context.deployment_dir.joinpath("data", "laconicd-config")
+    deployment_config_dir = context.deployment_dir.joinpath("data", "laconicd-config")
     copytree(config_dir_path, deployment_config_dir, dirs_exist_ok=True)
+    # If supplied, add the initial persistent peers to the config file
+    if extra_args[1]:
+        initial_persistent_peers = extra_args[1]
+        _insert_persistent_peers(context.command_context.cluster_context.options, deployment_config_dir, initial_persistent_peers)
     # Copy the data directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_data_dir = command_context.deployment_dir.joinpath("data", "laconicd-data")
+    deployment_data_dir = context.deployment_dir.joinpath("data", "laconicd-data")
     copytree(data_dir_path, deployment_data_dir, dirs_exist_ok=True)
 
 

app/deploy.py

@@ -140,8 +140,8 @@ def exec_operation(ctx, extra_args):
             print(f"Running compose exec {service_name} {command_to_exec}")
         try:
             ctx.obj.docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
-        except DockerException as error:
-            print(f"container command returned error exit status")
+        except DockerException:
+            print("container command returned error exit status")
 
 
 def logs_operation(ctx, tail: int, follow: bool, extra_args: str):

app/deploy_types.py

@@ -19,6 +19,7 @@ from pathlib import Path
 from python_on_whales import DockerClient
 from app.command_types import CommandOptions
 
+
 @dataclass
 class ClusterContext:
     options: CommandOptions  # TODO: this should be in its own object not stuffed in here
@@ -64,4 +65,4 @@ class LaconicStackSetupCommand:
 
 @dataclass
 class LaconicStackCreateCommand:
-    network_dir: str
+    network_dir: str

app/deploy_util.py

@@ -19,7 +19,7 @@ from app.deploy_types import DeployCommandContext, VolumeMapping
 from app.util import get_parsed_stack_config, get_yaml, get_compose_file_dir
 
 
-def _container_image_from_service(stack :str, service: str):
+def _container_image_from_service(stack: str, service: str):
     # Parse the compose files looking for the image name of the specified service
     image_name = None
     parsed_stack = get_parsed_stack_config(stack)

app/deployment.py

@@ -17,8 +17,8 @@ import click
 from dataclasses import dataclass
 from pathlib import Path
 import sys
-from app.deploy import up_operation, down_operation, ps_operation, port_operation, exec_operation, logs_operation, create_deploy_context
-from app.util import global_options
+from app.deploy import up_operation, down_operation, ps_operation, port_operation
+from app.deploy import exec_operation, logs_operation, create_deploy_context
 
 
 @dataclass

app/deployment_create.py

@@ -41,7 +41,7 @@ def _get_ports(stack):
             for svc_name, svc in parsed_pod_file["services"].items():
                 if "ports" in svc:
                     # Ports can appear as strings or numbers.  We normalize them as strings.
-                    ports[svc_name] = [ str(x) for x in svc["ports"] ]
+                    ports[svc_name] = [str(x) for x in svc["ports"]]
     return ports
 
 
@@ -178,7 +178,7 @@ def _get_mapped_ports(stack: str, map_recipe: str):
                     ports_array = ports[service]
                     for x in range(0, len(ports_array)):
                         orig_port = ports_array[x]
-                        random_port = random.randint(20000,50000) # Beware: we're relying on luck to not collide
+                        random_port = random.randint(20000, 50000)  # Beware: we're relying on luck to not collide
                         if map_recipe == "any-variable-random":
                             # This is the default so take no action
                             pass
@@ -202,11 +202,11 @@ def _get_mapped_ports(stack: str, map_recipe: str):
     return ports
 
 
-
 @click.command()
 @click.option("--output", required=True, help="Write yaml spec file here")
 @click.option("--map-ports-to-host", required=False,
-              help="Map ports to the host as one of: any-variable-random (default), localhost-same, any-same, localhost-fixed-random, any-fixed-random")
+              help="Map ports to the host as one of: any-variable-random (default), "
+              "localhost-same, any-same, localhost-fixed-random, any-fixed-random")
 @click.pass_context
 def init(ctx, output, map_ports_to_host):
     yaml = get_yaml()
@@ -238,8 +238,9 @@ def init(ctx, output, map_ports_to_host):
 @click.option("--deployment-dir", help="Create deployment files in this directory")
 # TODO: Hack
 @click.option("--network-dir", help="Network configuration supplied in this directory")
+@click.option("--initial-peers", help="Initial set of persistent peers")
 @click.pass_context
-def create(ctx, spec_file, deployment_dir, network_dir):
+def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
     # This function fails with a useful error message if the file doens't exist
     parsed_spec = get_parsed_deployment_spec(spec_file)
     stack_name = parsed_spec['stack']
@@ -287,7 +288,7 @@ def create(ctx, spec_file, deployment_dir, network_dir):
     deployment_command_context = ctx.obj
     deployment_command_context.stack = stack_name
     deployment_context = DeploymentContext(Path(deployment_dir), deployment_command_context)
-    call_stack_deploy_create(deployment_context, network_dir)
+    call_stack_deploy_create(deployment_context, [network_dir, initial_peers])
 
 
 # TODO: this code should be in the stack .py files but

app/setup_repositories.py

@@ -76,7 +76,7 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
     try:
         current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).active_branch.name
         is_branch = True
-    except TypeError as error:
+    except TypeError:
         # This means that the current ref is not a branch, so possibly a tag
         # Let's try to get the tag
         current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
@@ -96,7 +96,9 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
     repoName = repo_path.split("/")[-1]
     full_filesystem_repo_path = os.path.join(dev_root_path, repoName)
     is_present = os.path.isdir(full_filesystem_repo_path)
-    (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(full_filesystem_repo_path) if is_present else (None, None)
+    (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(
+        full_filesystem_repo_path
+        ) if is_present else (None, None)
     if not quiet:
         present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
             else 'Needs to be fetched'
@@ -116,7 +118,7 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
                         origin = git_repo.remotes.origin
                         origin.pull(progress=None if quiet else GitProgress())
                     else:
-                        print(f"skipping pull because this repo checked out a tag")
+                        print("skipping pull because this repo checked out a tag")
                 else:
                     print("(git pull skipped)")
     if not is_present:
@@ -143,7 +145,10 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
         branch_to_checkout = repo_branch
 
     if branch_to_checkout:
-        if current_repo_branch_or_tag is None or (current_repo_branch_or_tag and (current_repo_branch_or_tag != branch_to_checkout)):
+        if current_repo_branch_or_tag is None or (
+            current_repo_branch_or_tag and (
+                current_repo_branch_or_tag != branch_to_checkout)
+                ):
             if not quiet:
                 print(f"switching to branch {branch_to_checkout} in repo {repo_path}")
             git_repo = git.Repo(full_filesystem_repo_path)

app/version.py

@@ -16,6 +16,7 @@
 import click
 import importlib.resources
 
+
 @click.command()
 @click.pass_context
 def command(ctx):

cli.py

@@ -14,7 +14,6 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import click
-from dataclasses import dataclass
 
 from app.command_types import CommandOptions
 from app import setup_repositories