change imports

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -2,10 +2,7 @@ name: Fixturenet-Eth-Plugeth-Test
 
 on:
   push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+    branches: 'ci-test'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/fixturenet-eth-test.yml

@@ -2,10 +2,7 @@ name: Fixturenet-Eth-Test
 
 on:
   push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-test'
+    branches: 'ci-test'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/publish.yml

@@ -5,8 +5,6 @@ on:
     branches:
       - main
       - publish-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
 
 jobs:
   publish:

.gitea/workflows/test-deploy.yml

@@ -7,8 +7,6 @@ on:
     branches:
       - main
       - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/test.yml

@@ -7,8 +7,6 @@ on:
     branches:
       - main
       - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1 +0,0 @@
-Change this file to trigger running the fixturenet-eth-plugeth-test CI job

.gitea/workflows/triggers/fixturenet-eth-test

@@ -1 +0,0 @@
-Change this file to trigger running the fixturenet-eth-test CI job

app/base.py

@@ -15,8 +15,8 @@
 
 import os
 from abc import ABC, abstractmethod
-from app.deploy import get_stack_status
 from decouple import config
+from .deploy import get_stack_status
 
 
 def get_stack(config, stack):

app/build_containers.py

@@ -27,8 +27,8 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from app.util import include_exclude_check, get_parsed_stack_config
-from app.base import get_npm_registry_url
+from .util import include_exclude_check, get_parsed_stack_config
+from .base import get_npm_registry_url
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -67,7 +67,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         print('Dev root directory doesn\'t exist, creating')
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from app import data
+    from . import data
     with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
         all_containers = container_list_file.read().splitlines()
 

app/build_npms.py

@@ -25,8 +25,8 @@ from decouple import config
 import click
 import importlib.resources
 from python_on_whales import docker, DockerException
-from app.base import get_stack
-from app.util import include_exclude_check, get_parsed_stack_config
+from .base import get_stack
+from .util import include_exclude_check, get_parsed_stack_config
 
 builder_js_image_name = "cerc/builder-js:local"
 
@@ -83,7 +83,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         os.makedirs(build_root_path)
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from app import data
+    from . import data
     with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file:
         all_packages = package_list_file.read().splitlines()
 

app/cli.py

@@ -15,14 +15,14 @@
 
 import click
 
-from app.command_types import CommandOptions
-from app import setup_repositories
-from app import build_containers
-from app import build_npms
-from app import deploy
-from app import version
-from app import deployment
-from app import update
+from .command_types import CommandOptions
+from . import setup_repositories
+from . import build_containers
+from . import build_npms
+from . import deploy
+from . import version
+from . import deployment
+from . import update
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 

app/data/compose/docker-compose-fixturenet-optimism.yml

@@ -70,7 +70,6 @@ services:
     command: "/run-op-geth.sh"
     ports:
       - "0.0.0.0:8545:8545"
-      - "0.0.0.0:8546:8546"
     healthcheck:
       test: ["CMD", "nc", "-vz", "localhost:8545"]
       interval: 30s

app/data/compose/docker-compose-ponder.yml

@@ -1,16 +1,15 @@
 version: '3.7'
 
 services:
-  ponder-app-indexer:
-    hostname: ponder-app-indexer
+  ponder-app:
     restart: unless-stopped
     image: cerc/ponder:local
     working_dir: /app/examples/token-erc20
     environment:
       CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99}
       CERC_PONDER_RPC_URL_1: ${PONDER_RPC_URL_1:-http://nitro-reverse-payment-proxy:8081}
-      CERC_PONDER_NITRO_PK: ${CERC_PONDER_INDEXER_NITRO_PK:-58368d20ff12f17669c06158c21d885897aa56f9be430edc789614bf9851d53f}
-      CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_INDEXER_NITRO_CHAIN_PK:-fb1e9af328c283ca3e2486e7c24d13582b7912057d8b9542ff41503c85bc05c0}
+      CERC_PONDER_NITRO_PK: ${CERC_PONDER_NITRO_PK:-58368d20ff12f17669c06158c21d885897aa56f9be430edc789614bf9851d53f}
+      CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_NITRO_CHAIN_PK:-fb1e9af328c283ca3e2486e7c24d13582b7912057d8b9542ff41503c85bc05c0}
       CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546}
       CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR}
       CERC_UPSTREAM_NITRO_ADDRESS: ${CERC_UPSTREAM_NITRO_ADDRESS:-0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE}
@@ -19,45 +18,14 @@ services:
     command: ["bash", "./ponder-start.sh"]
     volumes:
       - ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh
-      - ../config/ponder/ponder.indexer.config.ts:/app/examples/token-erc20/ponder.config.ts
+      - ../config/ponder/ponder.config.ts:/app/examples/token-erc20/ponder.config.ts
       - peers_ids:/peers
       - nitro_deployment:/nitro
-      - ponder_indexer_nitro_data:/app/examples/token-erc20/.ponder/nitro-db
-    ports:
-      - "42070"
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-  ponder-app-watcher:
-    hostname: ponder-app-watcher
-    depends_on:
-      - ponder-app-indexer
-    restart: unless-stopped
-    image: cerc/ponder:local
-    working_dir: /app/examples/token-erc20
-    environment:
-      CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99}
-      CERC_PONDER_NITRO_PK: ${CERC_PONDER_WATCHER_NITRO_PK:-febb3b74b0b52d0976f6571d555f4ac8b91c308dfa25c7b58d1e6a7c3f50c781}
-      CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_WATCHER_NITRO_CHAIN_PK:-be4aa664815ea3bc3d63118649a733f6c96b243744310806ecb6d96359ab62cf}
-      CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546}
-      CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR}
-      CERC_INDEXER_GQL_ENDPOINT: ${CERC_INDEXER_GQL_ENDPOINT:-http://ponder-app-indexer:42070/graphql}
-      CERC_INDEXER_NITRO_ADDRESS: ${CERC_INDEXER_NITRO_ADDRESS:-0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d}
-      CERC_INDEXER_NITRO_PAY_AMOUNT: ${CERC_INDEXER_NITRO_PAY_AMOUNT:-50}
-    command: ["bash", "./ponder-start.sh"]
-    volumes:
-      - ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh
-      - ../config/ponder/ponder.watcher.config.ts:/app/examples/token-erc20/ponder.config.ts
-      - peers_ids:/peers
-      - nitro_deployment:/nitro
-      - ponder_watcher_nitro_data:/app/examples/token-erc20/.ponder/nitro-db
-    ports:
-      - "42069"
+      - ponder_nitro_data:/app/examples/token-erc20/.ponder/nitro-db
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
 volumes:
   peers_ids:
   nitro_deployment:
-  ponder_indexer_nitro_data:
-  ponder_watcher_nitro_data:
+  ponder_nitro_data:

app/data/config/ponder/ponder-start.sh

@@ -53,9 +53,6 @@ echo "RELAY_MULTIADDR=\"$CERC_RELAY_MULTIADDR\"" >> "$env_file"
 echo "UPSTREAM_NITRO_ADDRESS=\"$CERC_UPSTREAM_NITRO_ADDRESS\"" >> "$env_file"
 echo "UPSTREAM_NITRO_MULTIADDR=\"$CERC_UPSTREAM_NITRO_MULTIADDR\"" >> "$env_file"
 echo "UPSTREAM_NITRO_PAY_AMOUNT=\"$CERC_UPSTREAM_NITRO_PAY_AMOUNT\"" >> "$env_file"
-echo "INDEXER_GQL_ENDPOINT=\"$CERC_INDEXER_GQL_ENDPOINT\"" >> "$env_file"
-echo "INDEXER_NITRO_ADDRESS=\"$CERC_INDEXER_NITRO_ADDRESS\"" >> "$env_file"
-echo "INDEXER_NITRO_PAY_AMOUNT=\"$CERC_INDEXER_NITRO_PAY_AMOUNT\"" >> "$env_file"
 
 cat "$env_file"
 

app/data/config/ponder/ponder.config.ts

@@ -1,6 +1,6 @@
-import { type Config, AppMode } from "@ponder/core";
+import type { Config } from "@ponder/core";
 
-import contractAddresses from "./nitro-addresses.json" assert { type: "json" };
+import contractAddresses from "./nitro-addresses.json";
 
 export const config: Config = {
   networks: [
@@ -39,9 +39,6 @@ export const config: Config = {
       maxBlockRange: 100,
     },
   ],
-  options: {
-    mode: AppMode.Indexer,
-  },
   nitro: {
     privateKey: process.env.PONDER_NITRO_PK!,
     chainPrivateKey: process.env.PONDER_NITRO_CHAIN_PK!,

app/data/config/ponder/ponder.watcher.config.ts

@@ -1,46 +0,0 @@
-import { type Config, AppMode } from "@ponder/core";
-
-import contractAddresses from "./nitro-addresses.json" assert { type: "json" };
-
-export const config: Config = {
-  networks: [
-    {
-      name: "fixturenet",
-      chainId: Number(process.env.PONDER_CHAIN_ID),
-    },
-  ],
-  contracts: [
-    {
-      name: "AdventureGold",
-      network: "fixturenet",
-      abi: "./abis/AdventureGold.json",
-      address: "0x32353A6C91143bfd6C7d363B546e62a9A2489A20",
-      startBlock: 5,
-      maxBlockRange: 100,
-    },
-  ],
-  options: {
-    mode: AppMode.Watcher,
-  },
-  indexer: {
-    gqlEndpoint: process.env.INDEXER_GQL_ENDPOINT,
-    payments: {
-      nitro: {
-        address: process.env.INDEXER_NITRO_ADDRESS,
-        fundingAmounts: {
-          directFund: "1000000000000",
-          virtualFund: "1000000000",
-        },
-      },
-      amount: process.env.INDEXER_NITRO_PAY_AMOUNT,
-    },
-  },
-  nitro: {
-    privateKey: process.env.PONDER_NITRO_PK!,
-    chainPrivateKey: process.env.PONDER_NITRO_CHAIN_PK!,
-    chainUrl: process.env.PONDER_NITRO_CHAIN_URL!,
-    contractAddresses,
-    relayMultiAddr: process.env.RELAY_MULTIADDR!,
-    store: "./.ponder/nitro-db",
-  }
-};

app/data/container-build/cerc-plugeth-statediff/build.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
 # Build cerc/plugeth-statediff
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-# Pass Go auth token if present
-if [[ -n "${CERC_GO_AUTH_TOKEN}" ]]; then
-    build_command_args="${build_command_args} --build-arg GIT_VDBTO_TOKEN=${CERC_GO_AUTH_TOKEN}"
+# This container build currently requires access to private dependencies in gitea
+# so we check that the necessary access token has been supplied here, then pass it o the build
+if [[ -z "${CERC_GO_AUTH_TOKEN}" ]]; then
+    echo "ERROR: CERC_GO_AUTH_TOKEN is not set" >&2
+    exit 1
 fi
-docker build -t cerc/plugeth-statediff:local ${build_command_args} ${CERC_REPO_BASE_DIR}/plugeth-statediff
+docker build -t cerc/plugeth-statediff:local ${build_command_args} --build-arg GIT_VDBTO_TOKEN=${CERC_GO_AUTH_TOKEN} ${CERC_REPO_BASE_DIR}/plugeth-statediff

app/data/container-build/cerc-plugeth/build.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
 # Build cerc/plugeth
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-# Pass Go auth token if present
-if [[ -n "${CERC_GO_AUTH_TOKEN}" ]]; then
-    build_command_args="${build_command_args} --build-arg GIT_VDBTO_TOKEN=${CERC_GO_AUTH_TOKEN}"
+# This container build currently requires access to private dependencies in gitea
+# so we check that the necessary access token has been supplied here, then pass it o the build
+if [[ -z "${CERC_GO_AUTH_TOKEN}" ]]; then
+    echo "ERROR: CERC_GO_AUTH_TOKEN is not set" >&2
+    exit 1
 fi
-docker build -t cerc/plugeth:local ${build_command_args} ${CERC_REPO_BASE_DIR}/plugeth
+docker build -t cerc/plugeth:local ${build_command_args} --build-arg GIT_VDBTO_TOKEN=${CERC_GO_AUTH_TOKEN} ${CERC_REPO_BASE_DIR}/plugeth

app/data/container-image-list.txt

@@ -14,7 +14,6 @@ cerc/laconic-registry-cli
 cerc/laconic-console-host
 cerc/fixturenet-eth-geth
 cerc/fixturenet-eth-lighthouse
-cerc/fixturenet-eth-genesis
 cerc/watcher-ts
 cerc/watcher-mobymask
 cerc/watcher-erc20

app/data/stacks/fixturenet-payments/demo.md

@@ -204,15 +204,12 @@ Stack components:
 
 ### ERC20 Ponder App
 
-* Run the ponder app in indexer mode:
+* Run the ponder app in it's container:
 
   ```bash
-  docker exec -it payments-ponder-app-indexer-1 bash -c "DEBUG=laconic:payments pnpm start"
+  docker exec -it payments-ponder-app-1 bash -c "pnpm start"
 
   # Expected output:
-  # 08:00:28.701 INFO  payment    Nitro node setup with address 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d
-  #   laconic:payments Starting voucher subscription... +0ms
-  # ...
   # 09:58:54.288 INFO  payment    Creating ledger channel with nitro node 0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE
   # ...
   # 09:59:14.230 INFO  payment    Creating payment channel with nitro node 0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE
@@ -220,13 +217,7 @@ Stack components:
   # 09:59:14.329 INFO  payment    Using payment channel 0x10f049519bc3f862e2b26e974be8666886228f30ea54aab06e2f23718afffab0
   ```
 
-* Export the payment channel id to a variable:
-
-  ```bash
-  export PONDER_UPSTREAM_PAYMENT_CHANNEL=<PAYMENT_CHANNEL_ID>
-  ```
-
-* On starting the Ponder app in indexer mode, it creates a payment channel with the `ipld-eth-server`'s Nitro node and then starts the historical sync service
+* On starting the Ponder app, it creates a payment channel with the `ipld-eth-server`'s Nitro node and then starts the historical sync service
 
 * The sync service makes several ETH RPC requests to the `ipld-eth-server` to fetch required data; check the payment proxy server logs for charged RPC requests (`eth_getBlockByNumber`, `eth_getLogs`)
 
@@ -246,6 +237,12 @@ Stack components:
   # ...
   ```
 
+* Export the payment channel id to a variable:
+
+  ```bash
+  export PONDER_UPSTREAM_PAYMENT_CHANNEL=<PAYMENT_CHANNEL_ID>
+  ```
+
 * Check the ponder - ipld-eth-server payment channel status:
 
   ```bash
@@ -265,33 +262,6 @@ Stack components:
   # }
   ```
 
-* In another terminal run the ponder app in watcher mode:
-  ```bash
-  docker exec -it payments-ponder-app-watcher-1 bash -c "DEBUG=laconic:payments pnpm start"
-
-  # Expected output:
-  # 11:23:22.057 DEBUG app        Started using config file: ponder.config.ts
-  # 08:02:12.548 INFO  payment    Nitro node setup with address 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db
-  #   laconic:payments Starting voucher subscription... +0ms
-  # 08:02:17.417 INFO  payment    Creating ledger channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ...
-  # 08:02:37.135 INFO  payment    Creating payment channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ...
-  # 08:02:37.313 INFO  payment    Using payment channel 0x4b8e67f6a6fcfe114fdd60b85f963344ece4c77d4eea3825688c74b45ff5509b
-  # ...
-  # 11:23:22.436 INFO  server     Started responding as healthy
-  ```
-
-* Check the terminal in which indexer mode ponder is running. Logs of payment for `eth_getLogs` queries can be seen:
-  ```bash
-  # ...
-  # 08:02:37.763 DEBUG realtime   Finished processing new head block 89 (network=fixturenet)
-  #   laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +444ms
-  #   laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +1ms
-  # 08:02:37.804 DEBUG payment    Verified payment for GQL queries getLogEvents
-  #   laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +45ms
-  #   laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +0ms
-  # 08:02:37.849 DEBUG payment    Verified payment for GQL queries getLogEvents
-  ```
-  
 ## Clean Up
 
 * In the MobyMask app, perform `VIRTUAL DEFUND` and `DIRECT DEFUND` (in order) for closing the payment channel created with watcher
@@ -302,9 +272,4 @@ Stack components:
   await clearNodeStorage()
   ```
 
-* Run the following in the browser console to clear data in local storage:
-  ```bash
-  localStorage.clear()
-  ```
-
 * On a fresh restart, clear activity tab data in MetaMask for concerned accounts

app/data/stacks/fixturenet-payments/stack.yml

@@ -18,7 +18,7 @@ repos:
   - github.com/cerc-io/mobymask-snap
   - github.com/cerc-io/mobymask-ui@v0.2.1
   # ponder repo
-  - github.com/cerc-io/ponder@laconic-esm
+  - github.com/cerc-io/ponder@laconic
 containers:
   # fixturenet images
   - cerc/go-ethereum

app/data/stacks/fixturenet-plugeth-tx/README.md

@@ -12,7 +12,7 @@ See `stacks/fixturenet-eth/README.md` for more information.
 * cerc/tx-spammer
 
 ## Deploy the stack
-Note: if there are any private Go dependencies, `CERC_GO_AUTH_TOKEN` must be set to a valid Gitea access token before running the `build-containers` command.
+Note: since some Go dependencies are currently private, `CERC_GO_AUTH_TOKEN` must be set to a valid Gitea access token before running the `build-containers` command.
 ```
 $ laconic-so --stack fixturenet-plugeth-tx setup-repositories
 $ laconic-so --stack fixturenet-plugeth-tx build-containers

app/data/stacks/mainnet-laconic/deploy/commands.py

@@ -13,11 +13,11 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from app.util import get_yaml
-from app.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
-from app.stack_state import State
-from app.deploy_util import VolumeMapping, run_container_command
-from app.command_types import CommandOptions
+from laconic_stack_orchestrator.util import get_yaml
+from laconic_stack_orchestrator.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
+from laconic_stack_orchestrator.stack_state import State
+from laconic_stack_orchestrator.deploy_util import VolumeMapping, run_container_command
+from laconic_stack_orchestrator.command_types import CommandOptions
 from enum import Enum
 from pathlib import Path
 from shutil import copyfile, copytree

app/data/stacks/mobymask-v3/stack.yml

@@ -2,11 +2,11 @@ version: "1.0"
 description: "MobyMask v3 stack"
 name: mobymask-v3
 repos:
-  - github.com/cerc-io/ts-nitro@v0.1.13
-  - github.com/cerc-io/watcher-ts@v0.2.63
-  - github.com/cerc-io/mobymask-v2-watcher-ts@v0.2.2
+  - github.com/cerc-io/ts-nitrov0.1.12
+  - github.com/cerc-io/watcher-ts@v0.2.57
+  - github.com/cerc-io/mobymask-v2-watcher-ts@v3 # TODO: Update after fixes
   - github.com/cerc-io/MobyMask@v0.1.3
-  - github.com/cerc-io/mobymask-ui@v0.2.1
+  - github.com/cerc-io/mobymask-ui@v0.2.0
 containers:
   - cerc/nitro-contracts
   - cerc/watcher-ts

app/data/stacks/mobymask-v3/watcher.md

@@ -23,7 +23,7 @@ laconic-so --stack mobymask-v3 build-containers --exclude cerc/mobymask-ui
 Create and update an env file to be used in the next step ([defaults](../../config/watcher-mobymask-v3/mobymask-params.env)):
 
   ```bash
-  # External ETH RPC endpoint for contract(s) deployment
+  # External ETH RPC endpoint (L2 Optimism geth)
   CERC_ETH_RPC_ENDPOINT=
 
   # External ETH RPC endpoint used for queries in the watcher
@@ -32,9 +32,6 @@ Create and update an env file to be used in the next step ([defaults](../../conf
   # External ETH RPC endpoint used for mutations in the watcher
   CERC_ETH_RPC_MUTATION_ENDPOINT=
 
-  # External ETH endpoint used by watcher's Nitro node
-  CERC_NITRO_CHAIN_URL=
-
   # Specify the an account PK for contract deployment
   CERC_PRIVATE_KEY_DEPLOYER=
 

app/data/stacks/package-registry/stack.yml

@@ -2,7 +2,7 @@ version: "1.1"
 name: package-registry
 description: "Local Package Registry"
 repos:
-  - git.vdb.to/cerc-io/hosting
+  - github.com/cerc-io/hosting
   - gitea.com/gitea/act_runner
 containers:
   - cerc/act-runner

app/data/stacks/test/deploy/commands.py

@@ -13,10 +13,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from app.util import get_yaml
-from app.deploy_types import DeployCommandContext
-from app.stack_state import State
-from app.deploy_util import VolumeMapping, run_container_command
+from laconic_stack_orchestrator.util import get_yaml
+from laconic_stack_orchestrator.deploy_types import DeployCommandContext
+from laconic_stack_orchestrator.stack_state import State
+from laconic_stack_orchestrator.deploy_util import VolumeMapping, run_container_command
 from pathlib import Path
 
 default_spec_file_content = """config:

app/deploy.py

@@ -20,16 +20,17 @@ import copy
 import os
 import sys
 from dataclasses import dataclass
+from decouple import config
 from importlib import resources
 import subprocess
 from python_on_whales import DockerClient, DockerException
 import click
 from pathlib import Path
-from app.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
-from app.deploy_types import ClusterContext, DeployCommandContext
-from app.deployment_create import create as deployment_create
-from app.deployment_create import init as deployment_init
-from app.deployment_create import setup as deployment_setup
+from .util import include_exclude_check, get_parsed_stack_config, global_options2
+from .deploy_types import ClusterContext, DeployCommandContext
+from .deployment_create import create as deployment_create
+from .deployment_create import init as deployment_init
+from .deployment_create import setup as deployment_setup
 
 
 @click.group()
@@ -41,16 +42,9 @@ from app.deployment_create import setup as deployment_setup
 def command(ctx, include, exclude, env_file, cluster):
     '''deploy a stack'''
 
-    # Although in theory for some subcommands (e.g. deploy create) the stack can be inferred,
-    # Click doesn't allow us to know that here, so we make providing the stack mandatory
-    stack = global_options2(ctx).stack
-    if not stack:
-        print("Error: --stack option is required")
-        sys.exit(1)
-
     if ctx.parent.obj.debug:
         print(f"ctx.parent.obj: {ctx.parent.obj}")
-    ctx.obj = create_deploy_context(global_options2(ctx), stack, include, exclude, cluster, env_file)
+    ctx.obj = create_deploy_context(global_options2(ctx), global_options2(ctx).stack, include, exclude, cluster, env_file)
     # Subcommand is executed now, by the magic of click
 
 
@@ -241,15 +235,17 @@ def _make_runtime_env(ctx):
 # stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack
 def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
 
-    dev_root_path = get_dev_root_path(ctx)
+    if ctx.local_stack:
+        dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
+        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
+    else:
+        dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
     # TODO: huge hack, fix this
     # If the caller passed a path for the stack file, then we know that we can get the compose files
     # from the same directory
-    deployment = False
     if isinstance(stack, os.PathLike):
         compose_dir = stack.parent.joinpath("compose")
-        deployment = True
     else:
         # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
         compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose")
@@ -267,7 +263,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
             print(f"Using cluster name: {cluster}")
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from app import data
+    from . import data
     with resources.open_text(data, "pod-list.txt") as pod_list_file:
         all_pods = pod_list_file.read().splitlines()
 
@@ -300,24 +296,14 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
             if pod_repository is None or pod_repository == "internal":
                 compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
             else:
-                if deployment:
-                    compose_file_name = os.path.join(compose_dir, "docker-compose.yml")
-                    pod_pre_start_command = pod["pre_start_command"]
-                    pod_post_start_command = pod["post_start_command"]
-                    script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
-                    if pod_pre_start_command is not None:
-                        pre_start_commands.append(os.path.join(script_dir, pod_pre_start_command))
-                    if pod_post_start_command is not None:
-                        post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
-                else:
-                    pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
-                    compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml")
-                    pod_pre_start_command = pod["pre_start_command"]
-                    pod_post_start_command = pod["post_start_command"]
-                    if pod_pre_start_command is not None:
-                        pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
-                    if pod_post_start_command is not None:
-                        post_start_commands.append(os.path.join(pod_root_dir, pod_post_start_command))
+                pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
+                compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml")
+                pod_pre_start_command = pod["pre_start_command"]
+                pod_post_start_command = pod["post_start_command"]
+                if pod_pre_start_command is not None:
+                    pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
+                if pod_post_start_command is not None:
+                    post_start_commands.append(os.path.join(pod_root_dir, pod_post_start_command))
             compose_files.append(compose_file_name)
         else:
             if ctx.verbose:

app/deploy_types.py

@@ -17,7 +17,7 @@ from typing import List
 from dataclasses import dataclass
 from pathlib import Path
 from python_on_whales import DockerClient
-from app.command_types import CommandOptions
+from .command_types import CommandOptions
 
 
 @dataclass

app/deploy_util.py

@@ -15,15 +15,15 @@
 
 import os
 from typing import List
-from app.deploy_types import DeployCommandContext, VolumeMapping
-from app.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list
+from .deploy_types import DeployCommandContext, VolumeMapping
+from .util import get_parsed_stack_config, get_yaml, get_compose_file_dir
 
 
 def _container_image_from_service(stack: str, service: str):
     # Parse the compose files looking for the image name of the specified service
     image_name = None
     parsed_stack = get_parsed_stack_config(stack)
-    pods = get_pod_list(parsed_stack)
+    pods = parsed_stack["pods"]
     yaml = get_yaml()
     for pod in pods:
         pod_file_path = os.path.join(get_compose_file_dir(), f"docker-compose-{pod}.yml")

app/deployment.py

@@ -17,8 +17,8 @@ import click
 from dataclasses import dataclass
 from pathlib import Path
 import sys
-from app.deploy import up_operation, down_operation, ps_operation, port_operation
-from app.deploy import exec_operation, logs_operation, create_deploy_context
+from .deploy import up_operation, down_operation, ps_operation, port_operation
+from .deploy import exec_operation, logs_operation, create_deploy_context
 
 
 @dataclass

app/deployment_create.py

@@ -17,13 +17,12 @@ import click
 from importlib import util
 import os
 from pathlib import Path
-from typing import List
 import random
-from shutil import copy, copyfile, copytree
+from shutil import copyfile, copytree
 import sys
-from app.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml,
-                      get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_path)
-from app.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
+from .util import get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml
+from .util import get_compose_file_dir
+from .deploy_types import DeploymentContext, LaconicStackSetupCommand
 
 
 def _make_default_deployment_dir():
@@ -33,10 +32,10 @@ def _make_default_deployment_dir():
 def _get_ports(stack):
     ports = {}
     parsed_stack = get_parsed_stack_config(stack)
-    pods = get_pod_list(parsed_stack)
+    pods = parsed_stack["pods"]
     yaml = get_yaml()
     for pod in pods:
-        pod_file_path = get_pod_file_path(parsed_stack, pod)
+        pod_file_path = os.path.join(get_compose_file_dir(), f"docker-compose-{pod}.yml")
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
         if "services" in parsed_pod_file:
             for svc_name, svc in parsed_pod_file["services"].items():
@@ -50,10 +49,10 @@ def _get_named_volumes(stack):
     # Parse the compose files looking for named volumes
     named_volumes = []
     parsed_stack = get_parsed_stack_config(stack)
-    pods = get_pod_list(parsed_stack)
+    pods = parsed_stack["pods"]
     yaml = get_yaml()
     for pod in pods:
-        pod_file_path = get_pod_file_path(parsed_stack, pod)
+        pod_file_path = os.path.join(get_compose_file_dir(), f"docker-compose-{pod}.yml")
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
         if "volumes" in parsed_pod_file:
             volumes = parsed_pod_file["volumes"]
@@ -106,27 +105,16 @@ def _fixup_pod_file(pod, spec, compose_dir):
                 pod["services"][container_name]["ports"] = container_ports
 
 
-def _commands_plugin_path(ctx: DeployCommandContext):
-    plugin_path = get_plugin_code_path(ctx.stack)
-    return plugin_path.joinpath("deploy", "commands.py")
-
-
-# See: https://stackoverflow.com/a/54625079/1701505
-def _has_method(o, name):
-    return callable(getattr(o, name, None))
-
-
 def call_stack_deploy_init(deploy_command_context):
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_path = _commands_plugin_path(deploy_command_context)
+    python_file_path = get_stack_file_path(deploy_command_context.stack).parent.joinpath("deploy", "commands.py")
     if python_file_path.exists():
         spec = util.spec_from_file_location("commands", python_file_path)
         imported_stack = util.module_from_spec(spec)
         spec.loader.exec_module(imported_stack)
-        if _has_method(imported_stack, "init"):
-            return imported_stack.init(deploy_command_context)
+        return imported_stack.init(deploy_command_context)
     else:
         return None
 
@@ -136,14 +124,12 @@ def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetu
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_path = _commands_plugin_path(deploy_command_context)
-    print(f"Path: {python_file_path}")
+    python_file_path = get_stack_file_path(deploy_command_context.stack).parent.joinpath("deploy", "commands.py")
     if python_file_path.exists():
         spec = util.spec_from_file_location("commands", python_file_path)
         imported_stack = util.module_from_spec(spec)
         spec.loader.exec_module(imported_stack)
-        if _has_method(imported_stack, "setup"):
-            return imported_stack.setup(deploy_command_context, parameters, extra_args)
+        return imported_stack.setup(deploy_command_context, parameters, extra_args)
     else:
         return None
 
@@ -153,13 +139,12 @@ def call_stack_deploy_create(deployment_context, extra_args):
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_path = _commands_plugin_path(deployment_context.command_context)
+    python_file_path = get_stack_file_path(deployment_context.command_context.stack).parent.joinpath("deploy", "commands.py")
     if python_file_path.exists():
         spec = util.spec_from_file_location("commands", python_file_path)
         imported_stack = util.module_from_spec(spec)
         spec.loader.exec_module(imported_stack)
-        if _has_method(imported_stack, "create"):
-            return imported_stack.create(deployment_context, extra_args)
+        return imported_stack.create(deployment_context, extra_args)
     else:
         return None
 
@@ -278,21 +263,14 @@ def init(ctx, config, output, map_ports_to_host):
 
 def _write_config_file(spec_file: Path, config_env_file: Path):
     spec_content = get_parsed_deployment_spec(spec_file)
-    # Note: we want to write an empty file even if we have no config variables
-    with open(config_env_file, "w") as output_file:
-        if "config" in spec_content and spec_content["config"]:
-            config_vars = spec_content["config"]
-            if config_vars:
+    if spec_content["config"]:
+        config_vars = spec_content["config"]
+        if config_vars:
+            with open(config_env_file, "w") as output_file:
                 for variable_name, variable_value in config_vars.items():
                     output_file.write(f"{variable_name}={variable_value}\n")
 
 
-def _copy_files_to_directory(file_paths: List[Path], directory: Path):
-    for path in file_paths:
-        # Using copy to preserve the execute bit
-        copy(path, os.path.join(directory, os.path.basename(path)))
-
-
 @click.command()
 @click.option("--spec-file", required=True, help="Spec file to use to create this deployment")
 @click.option("--deployment-dir", help="Create deployment files in this directory")
@@ -320,19 +298,15 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
     # Copy any config varibles from the spec file into an env file suitable for compose
     _write_config_file(spec_file, os.path.join(deployment_dir, "config.env"))
     # Copy the pod files into the deployment dir, fixing up content
-    pods = get_pod_list(parsed_stack)
+    pods = parsed_stack['pods']
     destination_compose_dir = os.path.join(deployment_dir, "compose")
     os.mkdir(destination_compose_dir)
-    destination_pods_dir = os.path.join(deployment_dir, "pods")
-    os.mkdir(destination_pods_dir)
     data_dir = Path(__file__).absolute().parent.joinpath("data")
     yaml = get_yaml()
     for pod in pods:
-        pod_file_path = get_pod_file_path(parsed_stack, pod)
+        pod_file_path = os.path.join(get_compose_file_dir(), f"docker-compose-{pod}.yml")
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
         extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
-        destination_pod_dir = os.path.join(destination_pods_dir, pod)
-        os.mkdir(destination_pod_dir)
         if global_options(ctx).debug:
             print(f"extra config dirs: {extra_config_dirs}")
         _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
@@ -348,12 +322,6 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
                 # If the same config dir appears in multiple pods, it may already have been copied
                 if not os.path.exists(destination_config_dir):
                     copytree(source_config_dir, destination_config_dir)
-        # Copy the script files for the pod, if any
-        if pod_has_scripts(parsed_stack, pod):
-            destination_script_dir = os.path.join(destination_pod_dir, "scripts")
-            os.mkdir(destination_script_dir)
-            script_paths = get_pod_script_paths(parsed_stack, pod)
-            _copy_files_to_directory(script_paths, destination_script_dir)
     # Delegate to the stack's Python code
     # The deploy create command doesn't require a --stack argument so we need to insert the
     # stack member here.

app/setup_repositories.py

@@ -25,7 +25,7 @@ import click
 import importlib.resources
 from pathlib import Path
 import yaml
-from app.util import include_exclude_check
+from .util import include_exclude_check
 
 
 class GitProgress(git.RemoteProgress):
@@ -232,7 +232,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
         os.makedirs(dev_root_path)
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from app import data
+    from . import data
     with importlib.resources.open_text(data, "repository-list.txt") as repository_list_file:
         all_repos = repository_list_file.read().splitlines()
 

app/update.py

@@ -23,7 +23,7 @@ import sys
 import stat
 import shutil
 import validators
-from app.util import get_yaml
+from .util import get_yaml
 
 
 def _download_url(url: str, file_path: Path):

app/util.py

@@ -13,7 +13,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from decouple import config
 import os.path
 import sys
 import ruamel.yaml
@@ -38,16 +37,6 @@ def get_stack_file_path(stack):
     return stack_file_path
 
 
-def get_dev_root_path(ctx):
-    if ctx and ctx.local_stack:
-        # TODO: This code probably doesn't work
-        dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
-        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
-    else:
-        dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
-    return dev_root_path
-
-
 # Caller can pass either the name of a stack, or a path to a stack file
 def get_parsed_stack_config(stack):
     stack_file_path = stack if isinstance(stack, os.PathLike) else get_stack_file_path(stack)
@@ -67,68 +56,6 @@ def get_parsed_stack_config(stack):
         sys.exit(1)
 
 
-def get_pod_list(parsed_stack):
-    # Handle both old and new format
-    pods = parsed_stack["pods"]
-    if type(pods[0]) is str:
-        result = pods
-    else:
-        result = []
-        for pod in pods:
-            result.append(pod["name"])
-    return result
-
-
-def get_plugin_code_path(stack):
-    parsed_stack = get_parsed_stack_config(stack)
-    pods = parsed_stack["pods"]
-    # TODO: Hack
-    pod = pods[0]
-    if type(pod) is str:
-        result = get_stack_file_path(stack).parent
-    else:
-        pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
-        result = Path(os.path.join(pod_root_dir, "stack"))
-    return result
-
-
-def get_pod_file_path(parsed_stack, pod_name: str):
-    pods = parsed_stack["pods"]
-    if type(pods[0]) is str:
-        result = os.path.join(get_compose_file_dir(), f"docker-compose-{pod_name}.yml")
-    else:
-        for pod in pods:
-            if pod["name"] == pod_name:
-                pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
-                result = os.path.join(pod_root_dir, "docker-compose.yml")
-    return result
-
-
-def get_pod_script_paths(parsed_stack, pod_name: str):
-    pods = parsed_stack["pods"]
-    result = []
-    if not type(pods[0]) is str:
-        for pod in pods:
-            if pod["name"] == pod_name:
-                pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
-                if "pre_start_command" in pod:
-                    result.append(os.path.join(pod_root_dir, pod["pre_start_command"]))
-                if "post_start_command" in pod:
-                    result.append(os.path.join(pod_root_dir, pod["post_start_command"]))
-    return result
-
-
-def pod_has_scripts(parsed_stack, pod_name: str):
-    pods = parsed_stack["pods"]
-    if type(pods[0]) is str:
-        result = False
-    else:
-        for pod in pods:
-            if pod["name"] == pod_name:
-                result = "pre_start_command" in pod or "post_start_command" in pod
-    return result
-
-
 def get_compose_file_dir():
     # TODO: refactor to use common code with deploy command
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure

app/version.py

@@ -23,7 +23,7 @@ def command(ctx):
     '''print tool version'''
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from app import data
+    from . import data
     with importlib.resources.open_text(data, "build_tag.txt") as version_file:
         # TODO: code better version that skips comment lines
         version_string = version_file.read().splitlines()[1]

setup.py

@@ -1,5 +1,5 @@
 # See https://medium.com/nerd-for-tech/how-to-build-and-distribute-a-cli-tool-with-python-537ae41d9d78
-from setuptools import setup, find_packages
+from setuptools import setup
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 with open("requirements.txt", "r", encoding="utf-8") as fh:
@@ -14,8 +14,9 @@ setup(
     long_description=long_description,
     long_description_content_type="text/markdown",
     url='https://github.com/cerc-io/stack-orchestrator',
-    py_modules=['cli', 'app'],
-    packages=find_packages(),
+    packages=['laconic_stack_orchestrator'],
+    package_dir={'laconic_stack_orchestrator': 'app'},
+    py_modules=['laconic_stack_orchestrator', 'laconic_stack_orchestrator.cli'],
     install_requires=[requirements],
     python_requires='>=3.7',
     include_package_data=True,
@@ -25,6 +26,6 @@ setup(
         "Operating System :: OS Independent",
     ],
     entry_points={
-        'console_scripts': ['laconic-so=cli:cli'],
+        'console_scripts': ['laconic-so=laconic_stack_orchestrator.cli:cli'],
     }
 )

tests/deploy/run-deploy-test.sh

@@ -85,7 +85,7 @@ if [ ! -f "$test_deployment_spec" ]; then
     exit 1
 fi
 echo "deploy init test: passed"
-$TEST_TARGET_SO --stack test deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+$TEST_TARGET_SO deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
 # Check the deployment dir exists
 if [ ! -d "$test_deployment_dir" ]; then
     echo "deploy create test: deployment directory not present"

tests/smoke-test/run-smoke-test.sh

@@ -37,4 +37,8 @@ $TEST_TARGET_SO --stack test deploy-system down
 # Run same test but not using the stack definition
 # Test building the a stack container
 $TEST_TARGET_SO build-containers --include cerc/test-container
+# Deploy the test container
+$TEST_TARGET_SO deploy-system --include test up
+# Clean up
+$TEST_TARGET_SO deploy-system --include test down
 echo "Test passed"