Gitea x Laconicd + some doc updates (#549)

* combine instructions from 2 stacks


Former-commit-id: 77d194cfd9b0503c3e55e8c79429fa1f36a8d784

* tidy doc

---------

Co-authored-by: zramsay <zach@bluecollacoding.ca>

README.md

@@ -16,6 +16,7 @@ Ensure that the following are already installed:
 - [Python3](https://wiki.python.org/moin/BeginnersGuide/Download): `python3 --version` >= `3.8.10` (the Python3 shipped in Ubuntu 20+ is good to go)
 - [Docker](https://docs.docker.com/get-docker/): `docker --version` >= `20.10.21`
 - [jq](https://stedolan.github.io/jq/download/): `jq --version` >= `1.5`
+- [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git): `git --version` >= `2.10.3`
 
 Note: if installing docker-compose via package manager on Linux (as opposed to Docker Desktop), you must [install the plugin](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually), e.g. :
 
@@ -49,6 +50,13 @@ laconic-so version
 Version: 1.1.0-7a607c2-202304260513
 ```
 
+### Update
+If Stack Orchestrator was installed using the process described above, it is able to subsequently self-update to the current latest version by running:
+
+```bash
+laconic-so update
+```
+
 ## Usage
 
 The various [stacks](/app/data/stacks) each contain instructions for running different stacks based on your use case. For example:

app/update.py

@@ -0,0 +1,90 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import click
+import datetime
+import filecmp
+import os
+from pathlib import Path
+import requests
+import sys
+import stat
+import shutil
+import validators
+from app.util import get_yaml
+
+
+def _download_url(url: str, file_path: Path):
+    r = requests.get(url, stream=True)
+    r.raw.decode_content = True
+    with open(file_path, 'wb') as f:
+        shutil.copyfileobj(r.raw, f)
+
+
+def _error_exit(s: str):
+    print(s)
+    sys.exit(1)
+
+
+# Note at present this probably won't work on non-Unix based OSes like Windows
+@click.command()
+@click.option("--check-only", is_flag=True, default=False, help="only check, don't update")
+@click.pass_context
+def command(ctx, check_only):
+    '''update shiv binary from a distribution url'''
+    # Get the distribution URL from config
+    config_key = 'distribution-url'
+    config_file_path = Path(os.path.expanduser("~/.laconic-so/config.yml"))
+    if not config_file_path.exists():
+        _error_exit(f"Error: Config file: {config_file_path} not found")
+    yaml = get_yaml()
+    config = yaml.load(open(config_file_path, "r"))
+    if "distribution-url" not in config:
+        _error_exit(f"Error: {config_key} not defined in {config_file_path}")
+    distribution_url = config[config_key]
+    # Sanity check the URL
+    if not validators.url(distribution_url):
+        _error_exit(f"ERROR: distribution url: {distribution_url} is not valid")
+    # Figure out the filename for ourselves
+    shiv_binary_path = Path(sys.argv[0])
+    timestamp_filename = f"laconic-so-download-{datetime.datetime.now().strftime('%y%m%d-%H%M%S')}"
+    temp_download_path = shiv_binary_path.parent.joinpath(timestamp_filename)
+    # Download the file to a temp filename
+    if ctx.obj.verbose:
+        print(f"Downloading from: {distribution_url} to {temp_download_path}")
+    _download_url(distribution_url, temp_download_path)
+    # Set the executable bit
+    existing_mode = os.stat(temp_download_path)
+    os.chmod(temp_download_path, existing_mode.st_mode | stat.S_IXUSR)
+    # Switch the new file for the path we ran from
+    # Check if the downloaded file is identical to the existing one
+    same = filecmp.cmp(temp_download_path, shiv_binary_path)
+    if same:
+        if not ctx.obj.quiet or check_only:
+            print("No update available, latest version already installed")
+    else:
+        if not ctx.obj.quiet:
+            print("Update available")
+        if check_only:
+            if not ctx.obj.quiet:
+                print("Check-only node, update not installed")
+        else:
+            if not ctx.obj.quiet:
+                print("Installing...")
+            if ctx.obj.verbose:
+                print(f"Replacing: {shiv_binary_path} with {temp_download_path}")
+            os.replace(temp_download_path, shiv_binary_path)
+            if not ctx.obj.quiet:
+                print("Run \"laconic-so version\" to see the newly installed version")

cli.py

@@ -22,6 +22,7 @@ from app import build_npms
 from app import deploy
 from app import version
 from app import deployment
+from app import update
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
@@ -48,3 +49,4 @@ cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-syste
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")
 cli.add_command(version.command, "version")
+cli.add_command(update.command, "update")

docs/README.md

@@ -0,0 +1,8 @@
+# Stack Orchestrator
+
+Here you will find information about the design of stack orchestrator, contributing to it, and deploying services/applications that combine two or more "stacks".
+
+Most "stacks" contain their own README which has plenty of information on deploying, but stacks can be combined in a variety of ways which are document here, for example:
+
+- [Gitea with Laconicd Fixturenet](./gitea-with-laconicd-fixturenet.md)
+- [Laconicd Registry with Console](./laconicd-with-console.md)

docs/cli.md

@@ -6,7 +6,7 @@ Sub-commands and flags
 
 Clone a single repository:
 ```
-$ laconic-so setup-repositories --include cerc-io/go-ethereum
+$ laconic-so setup-repositories --include github.com/cerc-io/go-ethereum
 ```
 Clone the repositories for a stack:
 ```

docs/gitea-with-laconicd-fixturenet.md

@@ -0,0 +1,80 @@
+# Gitea x NPMs X Laconicd
+
+Deploy a local Gitea server, publish NPM packages to it, then use those packages to build a Laconicd fixturenet. Demonstrates several components of the Laconic stack
+
+### Build and Deploy Gitea
+
+```bash
+laconic-so --stack build-support build-containers
+laconic-so --stack package-registry setup-repositories
+laconic-so --stack package-registry build-containers 
+laconic-so --stack package-registry deploy up
+```
+
+These commands can take awhile. Eventually, some instructions and a token will output. Set `CERC_NPM_AUTH_TOKEN`:
+
+```bash
+export CERC_NPM_AUTH_TOKEN=<your-token>
+```
+
+### Configure the hostname gitea.local
+
+How to do this depends on your operating system  but usually involves editing a `hosts` file. For example, on Linux add this line to the file `/etc/hosts` (needs sudo):
+
+```bash
+127.0.0.1       gitea.local
+```
+
+Test with:
+
+```bash
+ping gitea.local
+```
+
+```bash
+PING gitea.local (127.0.0.1) 56(84) bytes of data.
+64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.147 ms
+64 bytes from localhost (127.0.0.1): icmp_seq=2 ttl=64 time=0.033 ms
+```
+
+Although not necessary in order to build and publish packages, you can now access the Gitea web interface at: [http://gitea.local:3000](http://gitea.local:3000) using these credentials: `gitea_admin/admin1234` (Note: please properly secure Gitea if public internet access is allowed).
+
+### Build npm Packages
+
+Clone the required repositories:
+
+```bash
+laconic-so --stack fixturenet-laconicd setup-repositories
+```
+
+Build and publish the npm packages:
+
+```bash
+laconic-so --stack fixturenet-laconicd build-npms
+```
+
+Navigate to the Gitea console and switch to the `cerc-io` user then find the `Packages` tab to confirm that these two npm packages have been published:
+
+- `@cerc-io/laconic-registry-cli`
+- `@cerc-io/laconic-sdk`
+
+### Build and deploy fixturenet containers
+
+```bash
+laconic-so --stack fixturenet-laconicd build-containers
+laconic-so --stack fixturenet-laconicd deploy up
+```
+
+Check the logs:
+
+```bash
+laconic-so --stack fixturenet-laconicd deploy logs
+```
+
+### Test with the registry CLI
+
+```bash
+laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
+```
+
+Try additional CLI commands, documented [here](https://github.com/cerc-io/laconic-registry-cli#operations).

docs/spec.md

@@ -1,7 +1,6 @@
 # Specification
 
-(note this page is out of date)
-
+Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://github.com/cerc-io/stack-orchestrator/issues/315).
 
 ## Implementation
 

requirements.txt

@@ -7,3 +7,4 @@ PyYAML>=6.0.1
 ruamel.yaml>=0.17.32
 pydantic==1.10.9
 tomli==2.0.1
+validators==0.22.0

scripts/quick-install-linux.sh

@@ -5,6 +5,9 @@ fi
 
 install_dir=~/bin
 
+# Skip the package install stuff if so directed
+if ! [[ -n "$CERC_SO_INSTALL_SKIP_PACKAGES" ]]; then
+
 # First display a reasonable warning to the user unless run with -y
 if ! [[ $# -eq 1 && $1 == "-y" ]]; then
   echo "**************************************************************************************"
@@ -128,13 +131,20 @@ sudo apt -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin d
 # Allow the current user to use Docker
 sudo usermod -aG docker $USER
 
+# End of long if block: Skip the package install stuff if so directed
+fi
+
 echo "**************************************************************************************"
 echo "Installing laconic-so"
 # install latest `laconic-so`
+distribution_url=https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
 install_filename=${install_dir}/laconic-so
 mkdir -p  ${install_dir}
-curl -L -o ${install_filename} https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+curl -L -o ${install_filename} ${distribution_url}
 chmod +x ${install_filename}
+# Set up config file for self-update feature
+mkdir ~/.laconic-so
+echo "distribution-url: ${distribution_url}" >  ~/.laconic-so/config.yml
 
 echo "**************************************************************************************"
 # Check if our PATH line is already there