squash me

debug

.gitea/workflows/gitea.yml

@@ -0,0 +1,51 @@
+name: Test Gitea
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - main
+      - zach-test
+      - new-gitea-test
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+jobs:
+  test:
+    name: "Run Gitea package registry test"
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Update'
+        run: apt-get update
+      - name: 'Setup jq'
+        run: apt-get install jq -y
+      - name: 'Check jq'
+        run: |
+          which jq
+          jq --version
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: "Run gitea tests"
+        run: |
+          export CERC_SCRIPT_DEBUG=true &
+          ./tests/gitea/test.sh
+

.github/workflows/gitea.yml

@@ -0,0 +1,29 @@
+name: Gitea Test
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+
+jobs:
+  test:
+    name: "Run basic test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run gitea tests"
+        run: ./tests/gitea/test.sh

README.md

@@ -16,6 +16,7 @@ Ensure that the following are already installed:
 - [Python3](https://wiki.python.org/moin/BeginnersGuide/Download): `python3 --version` >= `3.8.10` (the Python3 shipped in Ubuntu 20+ is good to go)
 - [Docker](https://docs.docker.com/get-docker/): `docker --version` >= `20.10.21`
 - [jq](https://stedolan.github.io/jq/download/): `jq --version` >= `1.5`
+- [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git): `git --version` >= `2.10.3`
 
 Note: if installing docker-compose via package manager on Linux (as opposed to Docker Desktop), you must [install the plugin](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually), e.g. :
 
@@ -49,6 +50,13 @@ laconic-so version
 Version: 1.1.0-7a607c2-202304260513
 ```
 
+### Update
+If Stack Orchestrator was installed using the process described above, it is able to subsequently self-update to the current latest version by running:
+
+```bash
+laconic-so update
+```
+
 ## Usage
 
 The various [stacks](/app/data/stacks) each contain instructions for running different stacks based on your use case. For example:

app/build_npms.py

@@ -132,8 +132,8 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
             try:
                 docker.run(builder_js_image_name,
                            remove=True,
-                           interactive=True,
-                           tty=True,
+                           interactive=False,
+                           tty=False,
                            user=f"{os.getuid()}:{os.getgid()}",
                            envs=envs,
                            # TODO: detect this host name in npm_registry_url rather than hard-wiring it

app/data/compose/docker-compose-fixturenet-plugeth.yml

@@ -20,7 +20,7 @@ services:
       - SYS_PTRACE
     environment:
       CERC_REMOTE_DEBUG: ${CERC_REMOTE_DEBUG:-true}
-      CERC_RUN_STATEDIFF: "detect"
+      CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect}
       CERC_STATEDIFF_DB_NODE_ID: 1
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     env_file:

app/data/compose/docker-compose-graph-node.yml

@@ -6,9 +6,6 @@ services:
         condition: service_healthy
       ipfs:
         condition: service_healthy
-      # Uncomment when running against fixturenet-lotus to wait for the Lotus node to come up
-      # lotus-node-1:
-      #   condition: service_healthy
     extra_hosts:
       - host.docker.internal:host-gateway
     environment:
@@ -18,9 +15,13 @@ services:
       postgres_user: graph-node
       postgres_pass: password
       postgres_db: graph-node
-      ethereum: ${NETWORK:-filecoin}:${ETH_RPC_ENDPOINT:-https://archive.lotus.vdb.to/rpc/v1}
+      ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
       GRAPH_LOG: debug
       ETHEREUM_REORG_THRESHOLD: 3
+    entrypoint: ["bash", "-c"]
+    # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
+    command: |
+      "wait_for ${ETH_RPC_HOST:-lotus-node-1}:${ETH_RPC_PORT:-1234} -t 1800 -- start"
     ports:
       - "8000"
       - "8001"

app/data/compose/docker-compose-mobymask-app-v3.yml

@@ -0,0 +1,39 @@
+version: '3.2'
+
+services:
+  # Builds and serves the MobyMask v3 react-app
+  mobymask-v3-app:
+    restart: unless-stopped
+    image: cerc/mobymask-ui:local
+    env_file:
+      - ../config/watcher-mobymask-v3/mobymask-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_CHAIN_ID: ${CERC_CHAIN_ID}
+      CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
+      CERC_RELAY_NODES: ${CERC_RELAY_NODES}
+      CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
+      CERC_PUBSUB: ${CERC_PUBSUB}
+      CERC_GOSSIPSUB_DIRECT_PEERS: ${CERC_GOSSIPSUB_DIRECT_PEERS}
+      CERC_RELEASE: "laconic-v3"
+      CERC_NA_ADDRESS: ${CERC_NA_ADDRESS}
+      CERC_VPA_ADDRESS: ${CERC_VPA_ADDRESS}
+      CERC_CA_ADDRESS: ${CERC_CA_ADDRESS}
+      CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
+      CERC_PAYMENT_NITRO_ADDRESS: ${CERC_PAYMENT_NITRO_ADDRESS}
+      CERC_SNAP_URL: ${CERC_SNAP_URL}
+    working_dir: /scripts
+    command: ["sh", "mobymask-app-start.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v3/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
+      - ../config/watcher-mobymask-v2/mobymask-app-config.json:/app/src/mobymask-app-config.json
+    ports:
+      - "127.0.0.1:3004:80"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "80"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"

app/data/compose/docker-compose-watcher-mobymask-v3.yml

@@ -0,0 +1,128 @@
+version: '3.2'
+
+services:
+  # Starts the PostgreSQL database for watcher
+  mobymask-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=mobymask-watcher,mobymask-watcher-job-queue
+      - POSTGRES_EXTENSION=mobymask-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - mobymask_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "127.0.0.1:15432:5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  # Deploys the MobyMask contract and generates an invite link
+  # Deployment is skipped if CERC_DEPLOYED_CONTRACT env is set
+  mobymask:
+    image: cerc/mobymask:local
+    working_dir: /app/packages/server
+    env_file:
+      - ../config/watcher-mobymask-v3/mobymask-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      ENV: "PROD"
+      CERC_L2_GETH_RPC: ${CERC_L2_GETH_RPC}
+      CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL}
+      CERC_PRIVATE_KEY_DEPLOYER: ${CERC_PRIVATE_KEY_DEPLOYER}
+      CERC_MOBYMASK_APP_BASE_URI: ${CERC_MOBYMASK_APP_BASE_URI}
+      CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
+      CERC_L2_GETH_HOST: ${CERC_L2_GETH_HOST}
+      CERC_L2_GETH_PORT: ${CERC_L2_GETH_PORT}
+      CERC_L2_NODE_HOST: ${CERC_L2_NODE_HOST}
+      CERC_L2_NODE_PORT: ${CERC_L2_NODE_PORT}
+    command: ["sh", "deploy-and-generate-invite.sh"]
+    volumes:
+      - ../config/network/wait-for-it.sh:/app/packages/server/wait-for-it.sh
+      - ../config/watcher-mobymask-v2/secrets-template.json:/app/packages/server/secrets-template.json
+      - ../config/watcher-mobymask-v2/deploy-and-generate-invite.sh:/app/packages/server/deploy-and-generate-invite.sh
+      - mobymask_deployment:/app/packages/server
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Creates peer-id files if they don't exist
+  peer-ids-gen:
+    image: cerc/watcher-ts:local
+    restart: on-failure
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+    working_dir: /app/packages/peer
+    command: ["sh", "generate-peer-ids.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v2/generate-peer-ids.sh:/app/packages/peer/generate-peer-ids.sh
+      - peers_ids:/peer-ids
+
+  # Optionally deploys the Nitro contracts; sets them at the required path
+  # Starts the MobyMask v3 watcher server
+  mobymask-watcher-server:
+    image: cerc/watcher-mobymask-v3:local
+    restart: unless-stopped
+    depends_on:
+      mobymask-watcher-db:
+        condition: service_healthy
+      peer-ids-gen:
+        condition: service_completed_successfully
+      mobymask:
+        condition: service_completed_successfully
+    env_file:
+      - ../config/watcher-mobymask-v3/mobymask-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_L2_GETH_RPC: ${CERC_L2_GETH_RPC}
+      CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL}
+      CERC_PRIVATE_KEY_DEPLOYER: ${CERC_PRIVATE_KEY_DEPLOYER}
+      CERC_RELAY_PEERS: ${CERC_RELAY_PEERS}
+      CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
+      CERC_PUBSUB: ${CERC_PUBSUB}
+      CERC_RELAY_ANNOUNCE_DOMAIN: ${CERC_RELAY_ANNOUNCE_DOMAIN}
+      CERC_ENABLE_PEER_L2_TXS: ${CERC_ENABLE_PEER_L2_TXS}
+      CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
+      CERC_NA_ADDRESS: ${CERC_NA_ADDRESS}
+      CERC_VPA_ADDRESS: ${CERC_VPA_ADDRESS}
+      CERC_CA_ADDRESS: ${CERC_CA_ADDRESS}
+      CERC_PRIVATE_KEY_PEER: ${CERC_PRIVATE_KEY_PEER}
+      CERC_PRIVATE_KEY_NITRO: ${CERC_PRIVATE_KEY_NITRO}
+      CERC_PEER_ID: ${CERC_PEER_ID}
+    entrypoint: ["bash", "-c"]
+    command: ["./deploy-nitro-contracts.sh && ./start-server.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v3/deploy-nitro-contracts.sh:/app/deploy-nitro-contracts.sh
+      - ../config/watcher-mobymask-v3/deploy-nitro-contracts.ts:/app/deploy-nitro-contracts.ts
+      - ../config/watcher-mobymask-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-mobymask-v3/watcher-config-rates.toml:/app/environments/rates.toml
+      - ../config/watcher-mobymask-v3/keys:/app/keys
+      - ../config/watcher-mobymask-v3/start-server.sh:/app/start-server.sh
+      - nitro_data:/app/out/nitro-db
+      - peers_ids:/app/peers
+      - nitro_addresses:/nitro
+      - mobymask_deployment:/server
+    # Expose GQL, metrics and relay node ports
+    ports:
+      - "127.0.0.1:3001:3001"
+      - "127.0.0.1:9001:9001"
+      - "127.0.0.1:9090:9090"
+    healthcheck:
+      test: ["CMD", "busybox", "nc", "localhost", "9090"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  mobymask_watcher_db_data:
+  peers_ids:
+  mobymask_deployment:
+  nitro_addresses:
+  nitro_data:

app/data/config/fixturenet-sushiswap-subgraph-v3/lotus-fixturenet.js.template

@@ -17,8 +17,8 @@ module.exports = {
     whitelistedTokenAddresses: [
       'NATIVE_ADDRESS',
     ],
-    stableTokenAddresses: [
-    ],
+    stableTokenAddresses: [],
+    nativePricePool: '0x0000000000000000000000000000000000000000',
     minimumEthLocked: 1.5
   }
 }

app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml

@@ -10,6 +10,13 @@ scrape_configs:
     static_configs:
     - targets: ['mainnet-eth-geth-1:6060']
 
+  # lighthouse
+  - job_name: 'lighthouse'
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+    - targets: ['mainnet-eth-lighthouse-1:5054']
+
   # keycloak
   - job_name: 'keycloak'
     scrape_interval: 5s

app/data/config/watcher-mobymask-v3/deploy-nitro-contracts.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+CERC_NA_ADDRESS="${CERC_NA_ADDRESS:-${DEFAULT_CERC_NA_ADDRESS}}"
+CERC_VPA_ADDRESS="${CERC_VPA_ADDRESS:-${DEFAULT_CERC_VPA_ADDRESS}}"
+CERC_CA_ADDRESS="${CERC_CA_ADDRESS:-${DEFAULT_CERC_CA_ADDRESS}}"
+
+NITRO_ADDRESSES_FILE_PATH="/nitro/nitro-addresses.json"
+
+# Check if CERC_NA_ADDRESS environment variable set to skip contract deployment
+if [ -n "$CERC_NA_ADDRESS" ]; then
+  echo "CERC_NA_ADDRESS is set to '$CERC_NA_ADDRESS'"
+  echo "CERC_VPA_ADDRESS is set to '$CERC_VPA_ADDRESS'"
+  echo "CERC_CA_ADDRESS is set to '$CERC_CA_ADDRESS'"
+  echo "Using the above addresses and skipping Nitro contracts deployment"
+
+  # Create the required JSON and write it to a file
+  nitro_addresses_json=$(jq -n \
+    --arg na "$CERC_NA_ADDRESS" \
+    --arg vpa "$CERC_VPA_ADDRESS" \
+    --arg ca "$CERC_CA_ADDRESS" \
+    '.nitroAdjudicatorAddress = $na | .virtualPaymentAppAddress = $vpa | .consensusAppAddress = $ca')
+  echo "$nitro_addresses_json" > "${NITRO_ADDRESSES_FILE_PATH}"
+
+  exit
+fi
+
+# Check and exit if a deployment already exists (on restarts)
+if [ -f ${NITRO_ADDRESSES_FILE_PATH} ]; then
+  echo "${NITRO_ADDRESSES_FILE_PATH} already exists, skipping Nitro contracts deployment"
+  exit
+fi
+
+echo "Using L2 RPC endpoint ${CERC_L2_GETH_RPC}"
+
+if [ -n "$CERC_L1_ACCOUNTS_CSV_URL" ] && \
+  l1_accounts_response=$(curl -L --write-out '%{http_code}' --silent --output /dev/null "$CERC_L1_ACCOUNTS_CSV_URL") && \
+  [ "$l1_accounts_response" -eq 200 ];
+then
+  echo "Fetching L1 account credentials using provided URL"
+  mkdir -p /geth-accounts
+  wget -O /geth-accounts/accounts.csv "$CERC_L1_ACCOUNTS_CSV_URL"
+
+  # Read the private key of an L1 account to deploy contract
+  CERC_PRIVATE_KEY_DEPLOYER=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 3)
+else
+  echo "Couldn't fetch L1 account credentials, using CERC_PRIVATE_KEY_DEPLOYER from env"
+fi
+
+echo "RPC_URL=${CERC_L2_GETH_RPC}" > .env
+echo "NITRO_ADDRESSES_FILE_PATH=${NITRO_ADDRESSES_FILE_PATH}" >> .env
+echo "PRIVATE_KEY=${CERC_PRIVATE_KEY_DEPLOYER}" >> .env
+
+yarn ts-node --esm deploy-nitro-contracts.ts

app/data/config/watcher-mobymask-v3/deploy-nitro-contracts.ts

@@ -0,0 +1,49 @@
+import 'dotenv/config';
+import fs from 'fs';
+import { providers, Wallet } from 'ethers';
+import { deployContracts } from '@cerc-io/nitro-util';
+
+async function main () {
+  const rpcURL = process.env.RPC_URL;
+  const addressesFilePath = process.env.NITRO_ADDRESSES_FILE_PATH;
+  const deployerKey = process.env.PRIVATE_KEY;
+
+  if (!rpcURL) {
+    console.log('RPC_URL not set, skipping deployment');
+    return;
+  }
+
+  if (!addressesFilePath) {
+    console.log('NITRO_ADDRESSES_FILE_PATH not set, skipping deployment');
+    return;
+  }
+
+  if (!deployerKey) {
+    console.log('PRIVATE_KEY not set, skipping deployment');
+    return;
+  }
+
+  const provider = new providers.JsonRpcProvider(process.env.RPC_URL);
+  const signer = new Wallet(deployerKey, provider);
+
+  const [
+    nitroAdjudicatorAddress,
+    virtualPaymentAppAddress,
+    consensusAppAddress
+  ] = await deployContracts(signer as any);
+
+  const output = {
+    nitroAdjudicatorAddress,
+    virtualPaymentAppAddress,
+    consensusAppAddress
+  };
+
+  fs.writeFileSync(addressesFilePath, JSON.stringify(output, null, 2));
+  console.log('Nitro contracts deployed, addresses written to', addressesFilePath);
+  console.log('Result:', JSON.stringify(output, null, 2));
+}
+
+main()
+  .catch((err) => {
+    console.log(err);
+  });

app/data/config/watcher-mobymask-v3/keys/12D3KooWAMjBkFCT9DtCnSDcxftxJzSuTBvzVojabv64cnEvX4AZ.json

@@ -0,0 +1,11 @@
+{
+  "peerId": {
+    "id": "12D3KooWAMjBkFCT9DtCnSDcxftxJzSuTBvzVojabv64cnEvX4AZ",
+    "privKey": "CAESQAKCrnY0QKTky1I18fqn+VPydXGUv1NYiV+nVKqBFkw/CAjE9sKKIDGnYAo8mivnI6dngFenERY+0Q8AJrPTaXY=",
+    "pubKey": "CAESIAgIxPbCiiAxp2AKPJor5yOnZ4BXpxEWPtEPACaz02l2"
+  },
+  "consensus": {
+    "publicKey": "02cd17b05ca998955be5ca7bf4fd4531243d438f1aae7ce8a0ed5159f53cee5b40",
+    "privateKey": "67d80505614bdf61fca11cbad31d93acb2c7df1c653dc25975d77d05f05f154f"
+  }
+}

app/data/config/watcher-mobymask-v3/keys/12D3KooWBNEbY3QS4y23ngupDw9PDc4bvNvRJGVRejjV9EZLjux5.json

@@ -0,0 +1,11 @@
+{
+  "peerId": {
+    "id": "12D3KooWBNEbY3QS4y23ngupDw9PDc4bvNvRJGVRejjV9EZLjux5",
+    "privKey": "CAESQGSTw0ymvn8+wX9Dbvyr4/Gib1q2voe0CC0VyeClMQP6FwW14x0fpRbBIx0XhLdxWHkRndphVg3gVAHyC+7ZI8o=",
+    "pubKey": "CAESIBcFteMdH6UWwSMdF4S3cVh5EZ3aYVYN4FQB8gvu2SPK"
+  },
+  "consensus": {
+    "publicKey": "029c8035b3e9401b8f178f7c37285b5cb22501e017340e2058b3b842f2a1f0ae45",
+    "privateKey": "0261008e8e3ec808168e99333599da38ca59a056a2ae4510a6ad3d8b5cb0918c"
+  }
+}

app/data/config/watcher-mobymask-v3/keys/12D3KooWSRH6ftgkAZsKZK7UX1Zr6Hx6YAsEepHqzopFszqfTxxi.json

@@ -0,0 +1,11 @@
+{
+  "peerId": {
+    "id": "12D3KooWSRH6ftgkAZsKZK7UX1Zr6Hx6YAsEepHqzopFszqfTxxi",
+    "privKey": "CAESQHBjlHxfVhZ2gXsBItrIEEgSGKcjMkFiGs3PPz9E3ace9qyWEkvR4oit5ve9SAROVoh20hoa42IC91NIafMaqws=",
+    "pubKey": "CAESIPaslhJL0eKIreb3vUgETlaIdtIaGuNiAvdTSGnzGqsL"
+  },
+  "consensus": {
+    "publicKey": "039160c244a7ad8be16a64bdb69e6dbacdcfe20b37076792a0d06032a8097468ca",
+    "privateKey": "8894685fe81001d75662b079905472699373967451d1255ee5fc669d0a09a9ca"
+  }
+}

app/data/config/watcher-mobymask-v3/mobymask-app-start.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+CERC_CHAIN_ID="${CERC_CHAIN_ID:-${DEFAULT_CERC_CHAIN_ID}}"
+CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRACT}}"
+CERC_RELAY_NODES="${CERC_RELAY_NODES:-${DEFAULT_CERC_RELAY_NODES}}"
+CERC_DENY_MULTIADDRS="${CERC_DENY_MULTIADDRS:-${DEFAULT_CERC_DENY_MULTIADDRS}}"
+CERC_PUBSUB="${CERC_PUBSUB:-${DEFAULT_CERC_PUBSUB}}"
+CERC_GOSSIPSUB_DIRECT_PEERS="${CERC_GOSSIPSUB_DIRECT_PEERS:-${DEFAULT_CERC_GOSSIPSUB_DIRECT_PEERS}}"
+
+echo "Using CERC_RELAY_NODES $CERC_RELAY_NODES"
+
+if [ -z "$CERC_DEPLOYED_CONTRACT" ]; then
+  echo "CERC_DEPLOYED_CONTRACT not set"
+  exit 1
+else
+  echo "Using CERC_DEPLOYED_CONTRACT ${CERC_DEPLOYED_CONTRACT} from env as the MobyMask contract address"
+fi
+
+# Checkout to the required release/branch
+cd /app
+git checkout $CERC_RELEASE
+
+# Check if CERC_NA_ADDRESS is set
+if [ -n "$CERC_NA_ADDRESS" ]; then
+  echo "CERC_NA_ADDRESS is set to '$CERC_NA_ADDRESS'"
+  echo "CERC_VPA_ADDRESS is set to '$CERC_VPA_ADDRESS'"
+  echo "CERC_CA_ADDRESS is set to '$CERC_CA_ADDRESS'"
+  echo "Using the above Nitro addresses"
+
+  # Create the required JSON and write it to a file
+  nitro_addresses_json=$(jq -n \
+    --arg na "$CERC_NA_ADDRESS" \
+    --arg vpa "$CERC_VPA_ADDRESS" \
+    --arg ca "$CERC_CA_ADDRESS" \
+    '.nitroAdjudicatorAddress = $na | .virtualPaymentAppAddress = $vpa | .consensusAppAddress = $ca')
+  echo "$nitro_addresses_json" > /app/src/utils/nitro-addresses.json
+else
+  echo "Nitro addresses not provided"
+  exit 1
+fi
+
+# Export config values in a json file
+jq --arg address "$CERC_DEPLOYED_CONTRACT" \
+  --argjson chainId "$CERC_CHAIN_ID" \
+  --argjson relayNodes "$CERC_RELAY_NODES" \
+  --argjson denyMultiaddrs "$CERC_DENY_MULTIADDRS" \
+  --arg pubsub "$CERC_PUBSUB" \
+  --argjson directPeers "$CERC_GOSSIPSUB_DIRECT_PEERS" \
+  '.address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.denyMultiaddrs = $denyMultiaddrs | .peer.pubsub = $pubsub | .peer.directPeers = $directPeers' \
+  /app/src/mobymask-app-config.json > /app/src/utils/config.json
+
+yarn install
+
+REACT_APP_WATCHER_URI="$CERC_APP_WATCHER_URL/graphql" \
+REACT_APP_PAY_TO_NITRO_ADDRESS="$CERC_PAYMENT_NITRO_ADDRESS" \
+REACT_APP_SNAP_ORIGIN="local:$CERC_SNAP_URL" \
+yarn build
+
+http-server -p 80 /app/build

app/data/config/watcher-mobymask-v3/mobymask-params.env

@@ -0,0 +1,37 @@
+# Defaults
+
+# Set of relay peers to connect to from the relay node
+DEFAULT_CERC_RELAY_PEERS=[]
+
+# Domain to be used in the relay node's announce address
+DEFAULT_CERC_RELAY_ANNOUNCE_DOMAIN=
+
+# Base URI for mobymask-app (used for generating invite)
+DEFAULT_CERC_MOBYMASK_APP_BASE_URI="http://127.0.0.1:3004/#"
+
+# Set to false for disabling watcher peer to send txs to L2
+DEFAULT_CERC_ENABLE_PEER_L2_TXS=true
+
+# Set deployed MobyMask contract address to avoid deploying contract in stack
+# mobymask-app will use this contract address in config if run separately
+DEFAULT_CERC_DEPLOYED_CONTRACT=
+
+# Chain ID is used by mobymask web-app for txs
+DEFAULT_CERC_CHAIN_ID=42069
+
+# Set of relay nodes to be used by web-apps
+DEFAULT_CERC_RELAY_NODES=[]
+
+# Set of multiaddrs to be avoided while dialling
+DEFAULT_CERC_DENY_MULTIADDRS=[]
+
+# Type of pubsub to be used
+DEFAULT_CERC_PUBSUB=""
+
+# Set of direct peers to be used when pubsub is set to gossipsub
+DEFAULT_CERC_GOSSIPSUB_DIRECT_PEERS=[]
+
+# Set deployed Nitro addresses to avoid deploying them in the stack
+DEFAULT_CERC_NA_ADDRESS=
+DEFAULT_CERC_VPA_ADDRESS=
+DEFAULT_CERC_CA_ADDRESS=

app/data/config/watcher-mobymask-v3/start-server.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+CERC_L2_GETH_RPC="${CERC_L2_GETH_RPC:-${DEFAULT_CERC_L2_GETH_RPC}}"
+
+CERC_RELAY_PEERS="${CERC_RELAY_PEERS:-${DEFAULT_CERC_RELAY_PEERS}}"
+CERC_DENY_MULTIADDRS="${CERC_DENY_MULTIADDRS:-${DEFAULT_CERC_DENY_MULTIADDRS}}"
+CERC_PUBSUB="${CERC_PUBSUB:-${DEFAULT_CERC_PUBSUB}}"
+CERC_RELAY_ANNOUNCE_DOMAIN="${CERC_RELAY_ANNOUNCE_DOMAIN:-${DEFAULT_CERC_RELAY_ANNOUNCE_DOMAIN}}"
+CERC_ENABLE_PEER_L2_TXS="${CERC_ENABLE_PEER_L2_TXS:-${DEFAULT_CERC_ENABLE_PEER_L2_TXS}}"
+CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRACT}}"
+
+nitro_addresses_file="/nitro/nitro-addresses.json"
+nitro_addresses_destination_file="./src/nitro-addresses.json"
+
+watcher_keys_dir="./keys"
+
+echo "Using L2 RPC endpoint ${CERC_L2_GETH_RPC}"
+
+# Use public domain for relay multiaddr in peer config if specified
+# Otherwise, use the docker container's host IP
+if [ -n "$CERC_RELAY_ANNOUNCE_DOMAIN" ]; then
+  CERC_RELAY_MULTIADDR="/dns4/${CERC_RELAY_ANNOUNCE_DOMAIN}/tcp/443/wss/p2p/$(jq -r '.id' /app/peers/relay-id.json)"
+else
+  CERC_RELAY_MULTIADDR="/dns4/mobymask-watcher-server/tcp/9090/ws/p2p/$(jq -r '.id' /app/peers/relay-id.json)"
+fi
+
+# Use contract address from environment variable or set from config.json in mounted volume
+if [ -n "$CERC_DEPLOYED_CONTRACT" ]; then
+  CONTRACT_ADDRESS="${CERC_DEPLOYED_CONTRACT}"
+else
+  # Assign deployed contract address from server config (created by mobymask container after deploying contract)
+  CONTRACT_ADDRESS=$(jq -r '.address' /server/config.json | tr -d '"')
+fi
+
+# Copy the deployed Nitro addresses to the required path
+if [ -f "$nitro_addresses_file" ]; then
+  cat "$nitro_addresses_file" > "$nitro_addresses_destination_file"
+  echo "Nitro addresses set to ${nitro_addresses_destination_file}"
+
+  # Build after setting the Nitro addresses
+  yarn build
+else
+  echo "File ${nitro_addresses_file} does not exist"
+  exit 1
+fi
+
+echo "Using CERC_PRIVATE_KEY_PEER (account with funds) from env for sending txs to L2"
+echo "Using CERC_PRIVATE_KEY_NITRO from env for Nitro account"
+
+if [ -n "$CERC_PEER_ID" ]; then
+  echo "Using CERC_PEER_ID ${CERC_PEER_ID} from env for watcher fixture"
+  echo "Consensus module enabled"
+
+  # Set corresponding variables
+  PEER_ID_FILE='./peer-id.json'
+  CONSENSUS_ENABLED=true
+  WATCHER_PARTY_PEERS_FILE='./watcher-party-peers.json'
+
+  # Create watcher party array
+  watcher_parties=()
+
+  # Iterate over each fixture JSON file
+  for peer_data_file in "$watcher_keys_dir"/*.json; do
+    # Extract the filename without the path and extension
+    peer_id=$(basename "$peer_data_file" .json)
+
+    # Read the consensus keys
+    consensus_public_key=$(jq -r '.consensus.publicKey' "$peer_data_file")
+    consensus_private_key=$(jq -r '.consensus.privateKey' "$peer_data_file")
+
+    # Append watcher party
+    watcher_party=$(jq -n \
+      --arg peerId "$peer_id" \
+      --arg publicKey "$consensus_public_key" \
+      '.peerId = $peerId | .publicKey = $publicKey')
+    watcher_parties+=("$watcher_party")
+
+    if [ "$peer_id" = "$CERC_PEER_ID" ]; then
+      # Export peer id
+      peer_id_data=$(jq '.peerId' "$peer_data_file")
+      echo "$peer_id_data" > "${PEER_ID_FILE}"
+
+      # Set consensus keys for this peer
+      CONSENSUS_PUBLIC_KEY=${consensus_public_key}
+      CONSENSUS_PRIVATE_KEY=${consensus_private_key}
+    fi
+  done
+
+  # Export watcher party file
+  watcher_parties_json=$(printf '%s\n' "${watcher_parties[@]}" | jq -s .)
+  echo "$watcher_parties_json" > "${WATCHER_PARTY_PEERS_FILE}"
+  echo "Watcher party peers exported to ${WATCHER_PARTY_PEERS_FILE}"
+else
+  echo "Using generated peer id"
+  echo "Consensus module disabled"
+
+  # Set corresponding variables
+  PEER_ID_FILE='./peers/peer-id.json'
+  CONSENSUS_ENABLED=false
+  WATCHER_PARTY_PEERS_FILE=''
+  CONSENSUS_PUBLIC_KEY=''
+  CONSENSUS_PRIVATE_KEY=''
+fi
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_RELAY_PEERS|${CERC_RELAY_PEERS}|g; \
+          s|REPLACE_WITH_CERC_DENY_MULTIADDRS|${CERC_DENY_MULTIADDRS}|g; \
+          s/REPLACE_WITH_CERC_PUBSUB/${CERC_PUBSUB}/g; \
+          s/REPLACE_WITH_CERC_RELAY_ANNOUNCE_DOMAIN/${CERC_RELAY_ANNOUNCE_DOMAIN}/g; \
+          s|REPLACE_WITH_CERC_RELAY_MULTIADDR|${CERC_RELAY_MULTIADDR}|g; \
+          s|REPLACE_WITH_PEER_ID_FILE|${PEER_ID_FILE}|g; \
+          s/REPLACE_WITH_CERC_ENABLE_PEER_L2_TXS/${CERC_ENABLE_PEER_L2_TXS}/g; \
+          s/REPLACE_WITH_CERC_PRIVATE_KEY_PEER/${CERC_PRIVATE_KEY_PEER}/g; \
+          s/REPLACE_WITH_CERC_PRIVATE_KEY_NITRO/${CERC_PRIVATE_KEY_NITRO}/g; \
+          s/REPLACE_WITH_CONTRACT_ADDRESS/${CONTRACT_ADDRESS}/g; \
+          s/REPLACE_WITH_CONSENSUS_ENABLED/${CONSENSUS_ENABLED}/g; \
+          s/REPLACE_WITH_CONSENSUS_PUBLIC_KEY/${CONSENSUS_PUBLIC_KEY}/g; \
+          s/REPLACE_WITH_CONSENSUS_PRIVATE_KEY/${CONSENSUS_PRIVATE_KEY}/g; \
+          s|REPLACE_WITH_WATCHER_PARTY_PEERS_FILE|${WATCHER_PARTY_PEERS_FILE}|g; \
+          s|REPLACE_WITH_CERC_L2_GETH_RPC_ENDPOINT|${CERC_L2_GETH_RPC}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo 'yarn server'
+yarn server

app/data/config/watcher-mobymask-v3/watcher-config-rates.toml

@@ -0,0 +1,14 @@
+freeQueriesLimit = 10
+
+freeQueriesList = []
+
+[queries]
+  multiNonce = '50'
+  _owner = '50'
+  isRevoked = '50'
+  isPhisher = '50'
+  isMember = '50'
+
+[mutations]
+  invoke = '100'
+  revoke = '100'

app/data/config/watcher-mobymask-v3/watcher-config-template.toml

@@ -0,0 +1,103 @@
+[server]
+  host = "0.0.0.0"
+  port = 3001
+  kind = "lazy"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  enableState = true
+
+  # Boolean to filter logs by contract.
+  filterLogs = true
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = -1
+
+  [server.p2p]
+    enableRelay = true
+    enablePeer = true
+
+    [server.p2p.relay]
+      host = "0.0.0.0"
+      port = 9090
+      relayPeers = REPLACE_WITH_CERC_RELAY_PEERS
+      denyMultiaddrs = REPLACE_WITH_CERC_DENY_MULTIADDRS
+      peerIdFile = './peers/relay-id.json'
+      announce = 'REPLACE_WITH_CERC_RELAY_ANNOUNCE_DOMAIN'
+      pubsub = 'REPLACE_WITH_CERC_PUBSUB'
+      enableDebugInfo = true
+
+    [server.p2p.peer]
+      relayMultiaddr = 'REPLACE_WITH_CERC_RELAY_MULTIADDR'
+      pubSubTopic = 'mobymask'
+      denyMultiaddrs = REPLACE_WITH_CERC_DENY_MULTIADDRS
+      peerIdFile = 'REPLACE_WITH_PEER_ID_FILE'
+      pubsub = 'REPLACE_WITH_CERC_PUBSUB'
+      enableDebugInfo = true
+      enableL2Txs = REPLACE_WITH_CERC_ENABLE_PEER_L2_TXS
+
+      [server.p2p.peer.l2TxsConfig]
+        privateKey = 'REPLACE_WITH_CERC_PRIVATE_KEY_PEER'
+        contractAddress = 'REPLACE_WITH_CONTRACT_ADDRESS'
+
+    [server.p2p.nitro]
+      store = './out/nitro-db'
+      privateKey = 'REPLACE_WITH_CERC_PRIVATE_KEY_NITRO'
+      chainPrivateKey = 'REPLACE_WITH_CERC_PRIVATE_KEY_PEER'
+
+      [server.p2p.nitro.payments]
+        ratesFile = './environments/rates.toml'
+        requestTimeoutInSecs = 10
+
+        [server.p2p.nitro.payments.cache]
+          maxAccounts = 1000
+          accountTTLInSecs = 1800
+          maxVouchersPerAccount = 1000
+          voucherTTLInSecs = 300
+          maxPaymentChannels = 10000
+          paymentChannelTTLInSecs = 1800
+
+    [server.p2p.consensus]
+      enabled = REPLACE_WITH_CONSENSUS_ENABLED
+      publicKey = 'REPLACE_WITH_CONSENSUS_PUBLIC_KEY'
+      privateKey = 'REPLACE_WITH_CONSENSUS_PRIVATE_KEY'
+      watcherPartyFile = 'REPLACE_WITH_WATCHER_PARTY_PEERS_FILE'
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "mobymask-watcher-db"
+  port = 5432
+  database = "mobymask-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server:8083/graphql"
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_L2_GETH_RPC_ENDPOINT"
+    blockDelayInMilliSecs = 60000
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@mobymask-watcher-db/mobymask-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50

app/data/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Usage: build-npm-package-local-dependencies.sh <registry-url> <publish-with-this-version>
 # Runs build-npm-package.sh after first fixing up yarn.lock to use a local
-# npm registry for all packages in a specific scope (currently @cerc-io, @lirewine and @muknsys)
+# npm registry for all packages in a specific scope (currently @cerc-io and @lirewine)
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
@@ -18,7 +18,7 @@ set -e
 local_npm_registry_url=$1
 package_publish_version=$2
 # If we need to handle an additional scope, add it to the list below:
-npm_scopes_to_handle=("@cerc-io" "@lirewine" "@muknsys")
+npm_scopes_to_handle=("@cerc-io" "@lirewine")
 for npm_scope_for_local in ${npm_scopes_to_handle[@]}
 do
     # We need to configure the local registry

app/data/container-build/cerc-builder-js/build-npm-package.sh

@@ -1,9 +1,25 @@
 #!/bin/bash
 # Usage: build-npm-package.sh <registry-url> <publish-with-this-version>
 # Note: supply the registry auth token in CERC_NPM_AUTH_TOKEN
+#
+
+
 if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
     set -x
 fi
+
+echo "GITEA ACTIONS 1 is: $GITEA_ACTIONS"
+echo "GITEA ACTIONS 2 is: ${GITEA_ACTIONS}"
+
+echo "token 1 is: $CERC_NPM_AUTH_TOKEN"
+echo "token 2 is: ${CERC_NPM_AUTH_TOKEN}"
+
+if [[ "$GITEA_ACTIONS" = "true" ]]; then
+	echo "WORKS"
+else
+	echo "WTF"
+fi
+
 if ! [[ $# -eq 1 || $# -eq 2 ]]; then
     echo "Illegal number of parameters" >&2
     exit 1
@@ -24,22 +40,24 @@ package_name=$( cat package.json | jq -r .name )
 local_npm_registry_url=$1
 npm config set @cerc-io:registry ${local_npm_registry_url}
 npm config set @lirewine:registry ${local_npm_registry_url}
-npm config set @muknsys:registry ${local_npm_registry_url}
 # Workaround bug in npm unpublish where it needs the url to be of the form //<foo> and not http://<foo>
 local_npm_registry_url_fixed=$( echo ${local_npm_registry_url} | sed -e 's/^http[s]\{0,1\}://')
 npm config set -- ${local_npm_registry_url_fixed}:_authToken ${CERC_NPM_AUTH_TOKEN}
 # First check if the version of this package we're trying to build already exists in the registry
+# but this line: "jq -r .data.dist.tarball" fails only in gitea, so skip this block
+if [[ "${GITEA_ACTIONS}" != "true" ]]; then
 package_exists=$( yarn info --json ${package_name}@${package_publish_version} 2>/dev/null | jq -r .data.dist.tarball )
-if [[ ! -z "$package_exists" && "$package_exists" != "null" ]]; then
-    echo "${package_publish_version} of ${package_name} already exists in the registry"
-    if [[ ${CERC_FORCE_REBUILD} == "true" ]]; then
-        # Attempt to unpublish the existing package
-        echo "NOTE: unpublishing existing package version since force rebuild is enabled"
-        npm unpublish --force ${package_name}@${package_publish_version}
-    else
-        echo "skipping build since target version already exists"
-        exit 0
-    fi
+  if [[ ! -z "$package_exists" && "$package_exists" != "null" ]]; then
+      echo "${package_publish_version} of ${package_name} already exists in the registry"
+      if [[ ${CERC_FORCE_REBUILD} == "true" ]]; then
+          # Attempt to unpublish the existing package
+          echo "NOTE: unpublishing existing package version since force rebuild is enabled"
+          npm unpublish --force ${package_name}@${package_publish_version}
+      else
+          echo "skipping build since target version already exists"
+          exit 0
+      fi
+  fi
 fi
 echo "Build and publish ${package_name} version ${package_publish_version}"
 yarn install

app/data/container-build/cerc-mobymask-ui/Dockerfile

@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=16-bullseye
+ARG VARIANT=18-bullseye
 FROM node:${VARIANT}
 
 ARG USERNAME=node
@@ -37,7 +37,7 @@ RUN yarn global add http-server
 
 WORKDIR /app
 COPY . .
-RUN npm install
+RUN yarn install
 
 # Expose port for http
 EXPOSE 80

app/data/container-build/cerc-watcher-erc20/build.sh

@@ -6,4 +6,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/watcher-erc20:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/erc20-watcher-ts
+docker build -t cerc/watcher-erc20:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/watcher-ts

app/data/container-build/cerc-watcher-mobymask-v3/Dockerfile

@@ -0,0 +1,20 @@
+FROM ubuntu:22.04
+
+RUN apt-get update \
+  && apt-get install -y curl wget gnupg build-essential \
+  && curl --silent --location https://deb.nodesource.com/setup_18.x | bash - \
+  && apt-get update \
+  && apt-get install -y nodejs git busybox jq \
+  && node -v
+
+RUN corepack enable \
+  && yarn --version
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building mobymask-v2-watcher-ts" && \
+  yarn && yarn build
+
+WORKDIR /app

app/data/container-build/cerc-watcher-mobymask-v3/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-mobymask-v3
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-mobymask-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mobymask-v2-watcher-ts

app/data/container-image-list.txt

@@ -50,3 +50,4 @@ cerc/watcher-sushiswap
 cerc/graph-node
 cerc/sushiswap-subgraphs
 cerc/webapp-base
+cerc/watcher-mobymask-v3

app/data/pod-list.txt

@@ -35,3 +35,5 @@ contract-sushiswap
 graph-node
 sushiswap-subgraph-v3
 fixturenet-sushiswap-subgraph-v3
+watcher-mobymask-v3
+mobymask-app-v3

app/data/stacks/erc20/stack.yml

@@ -4,7 +4,7 @@ repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/ipld-eth-db
   - github.com/cerc-io/ipld-eth-server
-  - github.com/cerc-io/erc20-watcher-ts
+  - github.com/cerc-io/watcher-ts
   - github.com/dboreham/foundry
 containers:
   - cerc/foundry

app/data/stacks/fixturenet-graph-node/README.md

@@ -1,3 +0,0 @@
-# Graph-Node Fixturenet
-
-Experimental

app/data/stacks/fixturenet-graph-node/stack.yml

@@ -1,12 +0,0 @@
-version: "1.0"
-name: fixturenet-graph-node
-description: "A graph-node fixturenet"
-repos:
-  - github.com/filecoin-project/lotus
-  - github.com/graphprotocol/graph-node
-containers:
-  - cerc/lotus
-  - cerc/graph-node
-pods:
-  - fixturenet-lotus
-  - graph-node

app/data/stacks/fixturenet-laconic-loaded/README.md

@@ -4,6 +4,8 @@ Testing a "Loaded" fixturenet with console.
 
 Instructions for deploying a local Laconic blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator.
 
+**Note:** For building some NPMs, access to the @lirewine repositories is required. If you don't have access, see [this tutorial](/docs/laconicd-fixturenet.md) to run this stack
+
 ## 1. Install Laconic Stack Orchestrator
 Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
 ```
@@ -30,24 +32,24 @@ $ export CERC_NPM_AUTH_TOKEN=<my-token>
 
 ## 3. Clone required repositories
 ```
-$ laconic-so --stack fixturenet-laconicd setup-repositories
+$ laconic-so --stack fixturenet-laconic-loaded setup-repositories
 ```
 ## 4. Build the stack's packages and containers
 ```
-$ laconic-so --stack fixturenet-laconicd build-npms
-$ laconic-so --stack fixturenet-laconicd build-containers
+$ laconic-so --stack fixturenet-laconic-loaded build-npms
+$ laconic-so --stack fixturenet-laconic-loaded build-containers
 ```
 ## 5. Deploy the stack
 ```
-$ laconic-so --stack fixturenet-laconicd deploy up
+$ laconic-so --stack fixturenet-laconic-loaded deploy up
 ```
 Correct operation should be verified by checking the laconicd container's logs with:
 ```
-$ laconic-so --stack fixturenet-laconicd deploy logs
+$ laconic-so --stack fixturenet-laconic-loaded deploy logs
 ```
 ## 6. Test with the Registry CLI
 ```
-$ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
+$ laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
 ```
 ## 7. View the laconic console
 Get the URL for the console web app with this command (the port number will be different for each deployment):

app/data/stacks/fixturenet-sushiswap-subgraph/README.md

@@ -32,21 +32,10 @@ laconic-so --stack fixturenet-sushiswap-subgraph build-containers
 
 ## Deploy
 
-
-Create an env file with the following contents to be used in the next step:
-
-```bash
-# Network and ETH RPC endpoint to run graph-node against
-NETWORK=lotus-fixturenet
-ETH_RPC_ENDPOINT=http://lotus-node-1:1234/rpc/v1
-```
-
-Uncomment the dependency on `lotus-node-1` in the [graph-node compose file](../../compose/docker-compose-graph-node.yml)
-
 Deploy the stack:
 
 ```bash
-laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph --env-file <PATH_TO_ENV_FILE> up
+laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph up
 
 # Note: Remove any existing volumes for the cluster for a fresh start
 ```
@@ -154,6 +143,42 @@ http://127.0.0.1:<HOST_PORT>/subgraphs/name/sushiswap/v3-lotus/graphql
   docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:burn:docker --pool $POOL_ADDRESS --amount 10
   ```
 
+* Query the sushiswap v3-lotus subgraph GQL after running above commands
+
+  ```graphql
+  {
+    _meta {
+      block {
+        number
+      }
+      deployment
+      hasIndexingErrors
+    }
+    
+    factories {
+      poolCount
+      id
+    }
+    
+    pools {
+      id
+      token0 {
+        id
+        name
+        symbol
+      }
+      mints {
+        id
+        owner
+      }
+      burns {
+        id
+        owner
+      }
+    }
+  }
+  ```
+
 ## Clean up
 
 Stop all the services running in background run:

app/data/stacks/graph-node/README.md

@@ -0,0 +1,157 @@
+# Graph Node
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack graph-node setup-repositories --pull
+```
+
+Checkout to a non-default branch in the cloned repos if required:
+
+```bash
+# Default repo base dir
+cd ~/cerc
+
+# Example
+cd graph-node
+git checkout <your-branch> && git pull
+
+# Remove the corresponding docker image if it already exists
+docker image rm cerc/graph-node:local
+# Remove any dangling images
+docker image prune
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack graph-node build-containers
+```
+
+## Create a deployment
+
+Initialize deployment and create "spec" file:
+
+```bash
+laconic-so --stack graph-node deploy init --output graph-node-spec.yml
+```
+
+We need to assign fixed ports: `8000` for subgraph GQL endpoint, `8020` for subgraph deployment and `5001` for IPFS. The values can be
+customized by editing the "spec" file generated by `laconic-so deploy init`.
+```
+$ cat graph-node-spec.yml
+stack: graph-node
+ports:
+  graph-node:
+    - '8000:8000'
+    - '8001'
+    - '8020:8020'
+    - '8030'
+  ipfs:
+    - '8080'
+    - '4001'
+    - '5001:5001'
+...
+```
+
+Create deployment:
+
+```bash
+laconic-so deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment
+```
+
+## Start the stack
+
+Create an env file with the following values to be set before starting the stack:
+
+```bash
+# Set ETH RPC endpoint the graph node will use
+
+# Host and port of the ETH RPC endpoint to check before starting graph-node
+export ETH_RPC_HOST=
+export ETH_RPC_PORT=
+
+# The etherum network(s) graph-node will connect to
+# Set this to a space-separated list of the networks where each entry has the form NAME:URL
+export ETH_NETWORKS=
+```
+
+Example env file:
+
+```bash
+export ETH_RPC_HOST=filecoin.chainup.net
+export ETH_RPC_PORT=443
+
+export ETH_NETWORKS=filecoin:https://filecoin.chainup.net/rpc/v1
+```
+
+Set the environment variables:
+
+```bash
+source <PATH_TO_ENV_FILE>
+```
+
+Deploy the stack:
+
+```bash
+laconic-so deployment --dir graph-node-deployment start
+
+# Note: Remove any existing volumes in the cluster for a fresh start
+```
+
+After all services have started, follow `graph-node` logs:
+
+```bash
+laconic-so deployment --dir graph-node-deployment logs -f graph-node
+```
+
+Subgraphs can now be deployed to the graph-node.
+Follow [Deploy the Subgraph](https://github.com/graphprotocol/graph-node/blob/v0.32.0/docs/getting-started.md#24-deploy-the-subgraph) section in graph-node docs for an existing subgraph.
+
+## Set environment variables
+
+* The graph-node environment variable `ETHEREUM_REORG_THRESHOLD` can be set in the deployment compose file
+  ```bash
+  $ cat graph-node-deployment/compose/docker-compose-graph-node.yml
+  services:
+    graph-node:
+      image: cerc/graph-node:local
+      ...
+      environment:
+        ...
+        GRAPH_LOG: debug
+        ETHEREUM_REORG_THRESHOLD: 3
+  ```
+  Change `ETHEREUM_REORG_THRESHOLD` to desired value
+
+  * To restart graph-node with updated values
+    * Stop the stack first
+      ```bash
+      laconic-so deployment --dir graph-node-deployment stop
+      ```
+    * Start the stack again
+      ```
+      laconic-so deployment --dir graph-node-deployment start
+      ```
+  * To check if environment variable has been updated in graph-node container
+      ```bash
+      $ laconic-so deployment --dir graph-node-deployment exec graph-node bash
+      root@dc4d3abe1615:/# echo $ETHEREUM_REORG_THRESHOLD
+      16
+      ```
+
+## Clean up
+
+Stop all the services running in background run:
+
+```bash
+laconic-so deployment --dir graph-node-deployment stop
+```
+
+Clear volumes created by this stack:
+
+```bash
+laconic-so deployment --dir graph-node-deployment stop --delete-volumes
+```

app/data/stacks/graph-node/deploy-subgraph.md

@@ -0,0 +1,68 @@
+# Deploying Subgraph
+
+## Setup
+
+We will use the [ethereum-gravatar](https://github.com/graphprotocol/graph-tooling/tree/%40graphprotocol/graph-cli%400.58.0/examples/ethereum-gravatar) example subgraph from `graphprotocol/graph-tooling` repo
+
+- Clone the repo
+  ```bash
+  git clone git@github.com:graphprotocol/graph-tooling.git
+  
+  cd graph-tooling
+  ```
+
+- Install dependencies
+  ```bash
+  pnpm install
+  ```
+
+- Change directory to example-subgraph
+  ```bash
+  cd examples/ethereum-gravatar
+  ```
+
+## Deploy
+
+The following steps should be similar for every subgraph
+
+- Change the network and address in `subgraph.yaml`
+  ```yaml
+  ...
+  dataSources:
+  - kind: ethereum/contract
+    name: Gravity
+    network: <NETWORK_NAME>
+    source:
+      address: '<CONTRACT_ADDRESS>'
+      abi: Gravity
+      startBlock: <START_BLOCK>
+  ...
+  ```
+  - `CONTRACT_ADDRESS` is the address of the deployed contract on the desired network
+  - `START_BLOCK` is the block number after which we want to process events
+  - `NETWORK_NAME` is the name of the network specified when deploying graph-node
+    - When deploying graph-node `ETH_NETWORKS` env is set to a space-separated list of the networks where each entry has the form `NAME:URL`
+    - The `NAME` can be used in subgraph to specify which network to use
+    - More details can be seen in [Start the stack](./README.md#start-the-stack) section
+
+- Build the subgraph
+  ```bash
+  pnpm codegen
+  pnpm build
+  ```
+
+- Create and deploy the subgraph
+  ```bash
+  pnpm graph create example --node <GRAPH_NODE_DEPLOY_ENDPOINT>
+  
+  pnpm graph deploy example --ipfs <GRAPH_NODE_IPFS_ENDPOINT> --node <GRAPH_NODE_DEPLOY_ENDPOINT>
+  ```
+  - `GRAPH_NODE_DEPLOY_ENDPOINT` and `GRAPH_NODE_IPFS_ENDPOINT` will be available after graph-node has been deployed
+  - More details can be seen in [Create a deployment](./README.md#create-a-deployment) section 
+
+- The subgraph GQL endpoint will be seen after deploy command runs successfully
+
+- To remove the subgraph
+  ```bash
+  pnpm graph remove --node <GRAPH_NODE_DEPLOY_ENDPOINT> example
+  ```

app/data/stacks/graph-node/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: graph-node
+description: "Stack for running graph-node"
+repos:
+  - github.com/graphprotocol/graph-node
+containers:
+  - cerc/graph-node
+pods:
+  - graph-node

app/data/stacks/mainnet-eth/README.md

@@ -17,7 +17,7 @@ $ laconic-so --stack mainnet-eth build-containers
 ## Create a deployment
 
 ```
-$ laconic-so --stack mainnet-eth deploy init --output mainnet-eth-spec.yml
+$ laconic-so --stack mainnet-eth deploy init --map-ports-to-host any-same --output mainnet-eth-spec.yml
 $ laconic-so deploy create --spec-file mainnet-eth-spec.yml --deployment-dir mainnet-eth-deployment
 ```
 ## Start the stack
@@ -138,4 +138,4 @@ $ sudo du -h mainnet-eth-deployment/data/
 860G    mainnet-eth-deployment/data/mainnet_eth_geth_1_data/geth
 860G    mainnet-eth-deployment/data/mainnet_eth_geth_1_data
 885G    mainnet-eth-deployment/data/
-```
+```

app/data/stacks/mainnet-eth/stack.yml

@@ -1,4 +1,4 @@
-version: "1.1"
+version: "1.2"
 name: mainnet-eth
 description: "Ethereum Mainnet"
 repos:
@@ -7,7 +7,6 @@ repos:
   - github.com/dboreham/foundry
   - git.vdb.to/cerc-io/keycloak-reg-api
   - git.vdb.to/cerc-io/keycloak-reg-ui
-  - github.com/vulcanize/eth-api-proxy
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
@@ -17,10 +16,8 @@ containers:
   - cerc/webapp-base
   - cerc/keycloak-reg-api
   - cerc/keycloak-reg-ui
-  - cerc/eth-api-proxy
 pods:
   - mainnet-eth
   - mainnet-eth-keycloak
   - mainnet-eth-metrics
-  - mainnet-eth-api-proxy
   - foundry

app/data/stacks/mobymask-v3/README.md

@@ -0,0 +1,6 @@
+# MobyMask v3
+
+Instructions to setup and deploy MobyMask v3 stack (watcher + web-app) using [laconic-stack-orchestrator](/README.md#install)
+
+* Follow [watcher.md](./watcher.md) for deploying the watcher
+* Follow [web-app.md](./web-app.md) for deploying the app

app/data/stacks/mobymask-v3/stack.yml

@@ -0,0 +1,16 @@
+version: "1.0"
+description: "MobyMask v3 stack"
+name: mobymask-v3
+repos:
+  - github.com/cerc-io/watcher-ts@v0.2.57
+  - github.com/cerc-io/mobymask-v2-watcher-ts@v3
+  - github.com/cerc-io/MobyMask@v0.1.3
+  - github.com/cerc-io/mobymask-ui
+containers:
+  - cerc/watcher-ts
+  - cerc/watcher-mobymask-v3
+  - cerc/mobymask
+  - cerc/mobymask-ui
+pods:
+  - watcher-mobymask-v3
+  - mobymask-app-v3

app/data/stacks/mobymask-v3/watcher.md

@@ -0,0 +1,135 @@
+# MobyMask v3 Watcher
+
+## Setup
+
+Prerequisite: L2 Optimism Geth and Node RPC endpoints
+
+Clone required repositories:
+
+```bash
+laconic-so --stack mobymask-v3 setup-repositories --pull --exclude github.com/cerc-io/mobymask-ui
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack mobymask-v3 build-containers --exclude cerc/mobymask-ui
+```
+
+## Deploy
+
+### Configuration
+
+Create and update an env file to be used in the next step ([defaults](../../config/watcher-mobymask-v3/mobymask-params.env)):
+
+  ```bash
+  # External L2 endpoints
+  CERC_L2_GETH_RPC=
+
+  # Endpoints waited on before contract deployment
+  CERC_L2_GETH_HOST=
+  CERC_L2_GETH_PORT=
+
+  CERC_L2_NODE_HOST=
+  CERC_L2_NODE_PORT=
+
+  # URL (fixturenet-eth-bootnode-lighthouse) to get CSV with credentials for accounts on L1 to perform txs on L2
+  CERC_L1_ACCOUNTS_CSV_URL=
+
+  # OR
+  # Specify the required account credentials
+  CERC_PRIVATE_KEY_DEPLOYER=
+
+  # Base URI for mobymask-app
+  # (used for generating a root invite link after deploying the contract)
+  CERC_MOBYMASK_APP_BASE_URI="http://127.0.0.1:3004/#"
+
+  # (Optional) Domain to be used in the relay node's announce address
+  CERC_RELAY_ANNOUNCE_DOMAIN=
+
+  # (Optional) Set of relay peers to connect to from the relay node
+  CERC_RELAY_PEERS=[]
+
+  # (Optional) Set of multiaddrs to be avoided while dialling
+  CERC_DENY_MULTIADDRS=[]
+
+  # (Optional) Type of pubsub to be used
+  CERC_PUBSUB=""
+
+  # Set to false for disabling watcher peer to send txs to L2
+  CERC_ENABLE_PEER_L2_TXS=true
+
+  # (Optional) Set already deployed MobyMask contract address to avoid deploying contract in the stack
+  CERC_DEPLOYED_CONTRACT=
+
+  # (Optional) Set already deployed Nitro addresses to avoid deploying them in the stack
+  CERC_NA_ADDRESS=
+  CERC_VPA_ADDRESS=
+  CERC_CA_ADDRESS=
+
+  # Specify private key of a funded account for sending txs to L2
+  CERC_PRIVATE_KEY_PEER=
+
+  # Specify private key for the Nitro account
+  CERC_PRIVATE_KEY_NITRO=
+
+  # (Optional) Set a pre-existing peer id to be used (enables consensus)
+  # Uses a generated peer id if not set (disables consensus)
+  CERC_PEER_ID=
+  ```
+
+* NOTE: If Optimism is running on the host machine, use `host.docker.internal` as the hostname to access the host port
+
+### Deploy the stack
+
+```bash
+laconic-so --stack mobymask-v3 deploy --cluster mobymask_v3 --include watcher-mobymask-v3 --env-file <PATH_TO_ENV_FILE> up
+```
+
+* To list down and monitor the running containers:
+
+  ```bash
+  laconic-so --stack mobymask-v3 deploy --cluster mobymask_v3 --include watcher-mobymask-v3 ps
+
+  # With status
+  docker ps -a
+
+  # Check logs for a container
+  docker logs -f <CONTAINER_ID>
+  ```
+
+* The watcher endpoint is exposed on host port `3001` and the relay node endpoint is exposed on host port `9090`
+
+* Check the logs of the MobyMask contract deployment container to get the deployed contract's address and generated root invite link:
+
+  ```bash
+  docker logs -f $(docker ps -aq --filter name="mobymask-1")
+  ```
+
+* Check the logs of the watcher server container to get the deployed Nitro contracts' addresses:
+
+```bash
+docker exec -it $(docker ps -q --filter name="mobymask-watcher-server") bash -c "cat /nitro/nitro-addresses.json"
+```
+
+## Clean up
+
+Stop all services running in the background:
+
+```bash
+laconic-so --stack mobymask-v3 deploy --cluster mobymask_v3 --include watcher-mobymask-v3 down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=mobymask_v3"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=mobymask_v3")
+
+# WARNING: To avoid changing peer ids for the watcher, `peers_ids` volume can be persisted
+# To delete all volumes except for `peers_ids`
+docker volume rm $(docker volume ls -q --filter "name=mobymask_v3" | grep -v "peers_ids$")
+```

app/data/stacks/mobymask-v3/web-app.md

@@ -0,0 +1,89 @@
+# MobyMask v3 App
+
+## Setup
+
+Prerequisite: Watcher with GQL and relay node endpoints
+
+Clone required repositories:
+
+```bash
+laconic-so --stack mobymask-v3 setup-repositories --pull --include github.com/cerc-io/mobymask-ui
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack mobymask-v3 build-containers --include cerc/mobymask-ui
+```
+
+## Deploy
+
+### Configuration
+
+Create and update an env file to be used in the next step ([defaults](../../config/watcher-mobymask-v3/mobymask-params.env)):
+
+  ```bash
+  # Set of relay nodes to be used by the web-app
+  # (use double quotes " for strings, avoid space after commas)
+  # Eg. CERC_RELAY_NODES=["/dns4/example.com/tcp/443/wss/p2p/12D3KooWGHmDDCc93XUWL16FMcTPCGu2zFaMkf67k8HZ4gdQbRDr"]
+  CERC_RELAY_NODES=[]
+
+  # Set of multiaddrs to be avoided while dialling
+  CERC_DENY_MULTIADDRS=[]
+
+  # Also add if running MobyMask app:
+
+  # Watcher endpoint used by the app for GQL queries
+  CERC_APP_WATCHER_URL="http://127.0.0.1:3001"
+
+  # Set deployed MobyMask contract address to be used in MobyMask app's config
+  CERC_DEPLOYED_CONTRACT=
+
+  # L2 Chain ID used by mobymask web-app for L2 txs
+  CERC_CHAIN_ID=42069
+
+  # (Optional) Type of pubsub to be used ("floodsub" | "gossipsub")
+  CERC_PUBSUB=""
+
+  # (Optional) Set of direct peers to be used when pubsub is set to gossipsub
+  CERC_GOSSIPSUB_DIRECT_PEERS=[]
+
+  # Set Nitro addresses
+  CERC_NA_ADDRESS=
+  CERC_VPA_ADDRESS=
+  CERC_CA_ADDRESS=
+
+  # Nitro account address to make the query and mutation payments to
+  CERC_PAYMENT_NITRO_ADDRESS=
+
+  # Endpoint for Mobymask snap installation
+  CERC_SNAP_URL=
+  ```
+
+### Deploy the stack
+
+```bash
+laconic-so --stack mobymask-v3 deploy --cluster mobymask_v3 --include mobymask-app-v3 --env-file <PATH_TO_ENV_FILE> up
+
+# Runs the MobyMask v3 app on host port 3004
+```
+
+To list down and monitor the running containers:
+
+```bash
+laconic-so --stack mobymask-v3 deploy --cluster mobymask_v3 --include mobymask-app-v3 ps
+
+# With status
+docker ps -a
+
+# Check logs for a container
+docker logs -f <CONTAINER_ID>
+```
+
+## Clean up
+
+Stop all services running in the background:
+
+```bash
+laconic-so --stack mobymask-v3 deploy --cluster mobymask_v3 --include mobymask-app-v3 down
+```

app/deployment_create.py

@@ -178,16 +178,18 @@ def _get_mapped_ports(stack: str, map_recipe: str):
                     ports_array = ports[service]
                     for x in range(0, len(ports_array)):
                         orig_port = ports_array[x]
+                        # Strip /udp suffix if present
+                        bare_orig_port = orig_port.replace("/udp", "")
                         random_port = random.randint(20000, 50000)  # Beware: we're relying on luck to not collide
                         if map_recipe == "any-variable-random":
                             # This is the default so take no action
                             pass
                         elif map_recipe == "localhost-same":
                             # Replace instances of "- XX" with "- 127.0.0.1:XX"
-                            ports_array[x] = f"127.0.0.1:{orig_port}:{orig_port}"
+                            ports_array[x] = f"127.0.0.1:{bare_orig_port}:{orig_port}"
                         elif map_recipe == "any-same":
                             # Replace instances of "- XX" with "- 0.0.0.0:XX"
-                            ports_array[x] = f"0.0.0.0:{orig_port}:{orig_port}"
+                            ports_array[x] = f"0.0.0.0:{bare_orig_port}:{orig_port}"
                         elif map_recipe == "localhost-fixed-random":
                             # Replace instances of "- XX" with "- 127.0.0.1:<rnd>:XX"
                             ports_array[x] = f"127.0.0.1:{random_port}:{orig_port}"

app/update.py

@@ -0,0 +1,90 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import click
+import datetime
+import filecmp
+import os
+from pathlib import Path
+import requests
+import sys
+import stat
+import shutil
+import validators
+from app.util import get_yaml
+
+
+def _download_url(url: str, file_path: Path):
+    r = requests.get(url, stream=True)
+    r.raw.decode_content = True
+    with open(file_path, 'wb') as f:
+        shutil.copyfileobj(r.raw, f)
+
+
+def _error_exit(s: str):
+    print(s)
+    sys.exit(1)
+
+
+# Note at present this probably won't work on non-Unix based OSes like Windows
+@click.command()
+@click.option("--check-only", is_flag=True, default=False, help="only check, don't update")
+@click.pass_context
+def command(ctx, check_only):
+    '''update shiv binary from a distribution url'''
+    # Get the distribution URL from config
+    config_key = 'distribution-url'
+    config_file_path = Path(os.path.expanduser("~/.laconic-so/config.yml"))
+    if not config_file_path.exists():
+        _error_exit(f"Error: Config file: {config_file_path} not found")
+    yaml = get_yaml()
+    config = yaml.load(open(config_file_path, "r"))
+    if "distribution-url" not in config:
+        _error_exit(f"Error: {config_key} not defined in {config_file_path}")
+    distribution_url = config[config_key]
+    # Sanity check the URL
+    if not validators.url(distribution_url):
+        _error_exit(f"ERROR: distribution url: {distribution_url} is not valid")
+    # Figure out the filename for ourselves
+    shiv_binary_path = Path(sys.argv[0])
+    timestamp_filename = f"laconic-so-download-{datetime.datetime.now().strftime('%y%m%d-%H%M%S')}"
+    temp_download_path = shiv_binary_path.parent.joinpath(timestamp_filename)
+    # Download the file to a temp filename
+    if ctx.obj.verbose:
+        print(f"Downloading from: {distribution_url} to {temp_download_path}")
+    _download_url(distribution_url, temp_download_path)
+    # Set the executable bit
+    existing_mode = os.stat(temp_download_path)
+    os.chmod(temp_download_path, existing_mode.st_mode | stat.S_IXUSR)
+    # Switch the new file for the path we ran from
+    # Check if the downloaded file is identical to the existing one
+    same = filecmp.cmp(temp_download_path, shiv_binary_path)
+    if same:
+        if not ctx.obj.quiet or check_only:
+            print("No update available, latest version already installed")
+    else:
+        if not ctx.obj.quiet:
+            print("Update available")
+        if check_only:
+            if not ctx.obj.quiet:
+                print("Check-only node, update not installed")
+        else:
+            if not ctx.obj.quiet:
+                print("Installing...")
+            if ctx.obj.verbose:
+                print(f"Replacing: {shiv_binary_path} with {temp_download_path}")
+            os.replace(temp_download_path, shiv_binary_path)
+            if not ctx.obj.quiet:
+                print("Run \"laconic-so version\" to see the newly installed version")

cli.py

@@ -22,6 +22,7 @@ from app import build_npms
 from app import deploy
 from app import version
 from app import deployment
+from app import update
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
@@ -48,3 +49,4 @@ cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-syste
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")
 cli.add_command(version.command, "version")
+cli.add_command(update.command, "update")

requirements.txt

@@ -7,3 +7,4 @@ PyYAML>=6.0.1
 ruamel.yaml>=0.17.32
 pydantic==1.10.9
 tomli==2.0.1
+validators==0.22.0

scripts/quick-install-linux.sh

@@ -5,6 +5,9 @@ fi
 
 install_dir=~/bin
 
+# Skip the package install stuff if so directed
+if ! [[ -n "$CERC_SO_INSTALL_SKIP_PACKAGES" ]]; then
+
 # First display a reasonable warning to the user unless run with -y
 if ! [[ $# -eq 1 && $1 == "-y" ]]; then
   echo "**************************************************************************************"
@@ -128,13 +131,20 @@ sudo apt -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin d
 # Allow the current user to use Docker
 sudo usermod -aG docker $USER
 
+# End of long if block: Skip the package install stuff if so directed
+fi
+
 echo "**************************************************************************************"
 echo "Installing laconic-so"
 # install latest `laconic-so`
+distribution_url=https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
 install_filename=${install_dir}/laconic-so
 mkdir -p  ${install_dir}
-curl -L -o ${install_filename} https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+curl -L -o ${install_filename} ${distribution_url}
 chmod +x ${install_filename}
+# Set up config file for self-update feature
+mkdir ~/.laconic-so
+echo "distribution-url: ${distribution_url}" >  ~/.laconic-so/config.yml
 
 echo "**************************************************************************************"
 # Check if our PATH line is already there

setup.py

@@ -19,8 +19,7 @@ setup(
     install_requires=[requirements],
     python_requires='>=3.7',
     include_package_data=True,
-    # See: https://github.com/pypa/setuptools/issues/1806
-    package_data={'': ['data/*', 'data/*/*', 'data/*/*/*', 'data/*/*/*/*', 'data/*/*/*/*/*']},
+    package_data={'': ['data/**']},
     classifiers=[
         "Programming Language :: Python :: 3.8",
         "Operating System :: OS Independent",

tests/gitea/test.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -e
+
+export CERC_SCRIPT_DEBUG=true
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+# Dump environment variables for debugging
+echo "Environment variables:"
+env
+
+echo "holla 1 $GITEA_ACTIONS"
+echo "Running gitea test"
+
+# Bit of a hack, test the most recent package
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+# Set a non-default repo dir
+export CERC_REPO_BASE_DIR=~/stack-orchestrator-gitea-test
+echo "Testing this package: $TEST_TARGET_SO"
+echo "Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "Version reported is: ${reported_version_string}"
+echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
+rm -rf $CERC_REPO_BASE_DIR
+mkdir -p $CERC_REPO_BASE_DIR
+
+$TEST_TARGET_SO --stack build-support build-containers
+$TEST_TARGET_SO --stack package-registry setup-repositories
+$TEST_TARGET_SO --stack package-registry build-containers
+output=$($TEST_TARGET_SO --stack package-registry deploy-system up)
+token=$(echo $output | grep -o 'export CERC_NPM_AUTH_TOKEN=[^ ]*' | cut -d '=' -f 2)
+export CERC_NPM_AUTH_TOKEN=$token
+export GITEA_ACTIONS=true
+echo "holla 2 $GITEA_ACTIONS"
+$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories 
+$TEST_TARGET_SO --stack fixturenet-laconicd --verbose --debug build-npms --include laconic-sdk
+
+# Clean up
+$TEST_TARGET_SO --stack package-registry deploy-system down
+echo "Test passed"