diff --git a/.gitea/workflows/test-deploy.yml b/.gitea/workflows/test-deploy.yml index 2ea72c08..b0d5194d 100644 --- a/.gitea/workflows/test-deploy.yml +++ b/.gitea/workflows/test-deploy.yml @@ -2,7 +2,8 @@ name: Deploy Test on: pull_request: - branches: '*' + branches: + - main push: branches: - main diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index a9964b72..bbd1d508 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -2,7 +2,8 @@ name: K8s Deploy Test on: pull_request: - branches: '*' + branches: + - main push: branches: '*' paths: diff --git a/.gitea/workflows/test-k8s-deployment-control.yml b/.gitea/workflows/test-k8s-deployment-control.yml index 9ab2526d..3784451b 100644 --- a/.gitea/workflows/test-k8s-deployment-control.yml +++ b/.gitea/workflows/test-k8s-deployment-control.yml @@ -2,7 +2,8 @@ name: K8s Deployment Control Test on: pull_request: - branches: '*' + branches: + - main push: branches: '*' paths: diff --git a/.gitea/workflows/test-webapp.yml b/.gitea/workflows/test-webapp.yml index 99c5138f..8a3a60f9 100644 --- a/.gitea/workflows/test-webapp.yml +++ b/.gitea/workflows/test-webapp.yml @@ -2,7 +2,8 @@ name: Webapp Test on: pull_request: - branches: '*' + branches: + - main push: branches: - main diff --git a/.gitea/workflows/triggers/test-container-registry b/.gitea/workflows/triggers/test-container-registry index e5c11f22..0cc5eb23 100644 --- a/.gitea/workflows/triggers/test-container-registry +++ b/.gitea/workflows/triggers/test-container-registry @@ -1 +1,3 @@ -Change this file to trigger running the test-container-registry CI job \ No newline at end of file +Change this file to trigger running the test-container-registry CI job +Triggered: 2026-01-21 +Triggered: 2026-01-21 19:28:29 diff --git a/.gitea/workflows/triggers/test-database b/.gitea/workflows/triggers/test-database index 2d1d61d2..f867b40b 100644 --- a/.gitea/workflows/triggers/test-database +++ b/.gitea/workflows/triggers/test-database @@ -1,2 +1,2 @@ Change this file to trigger running the test-database CI job -Trigger test run \ No newline at end of file +Trigger test run diff --git a/.github/workflows/triggers/fixturenet-eth-test b/.github/workflows/triggers/fixturenet-eth-test index 7d12834e..c95c9a8d 100644 --- a/.github/workflows/triggers/fixturenet-eth-test +++ b/.github/workflows/triggers/fixturenet-eth-test @@ -1,2 +1 @@ Change this file to trigger running the fixturenet-eth-test CI job - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..b01ad2be --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + args: ['--allow-multiple-documents'] + - id: check-json + - id: check-merge-conflict + - id: check-added-large-files + + - repo: https://github.com/psf/black + rev: 23.12.1 + hooks: + - id: black + language_version: python3 + + - repo: https://github.com/PyCQA/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + args: ['--max-line-length=88', '--extend-ignore=E203,W503,E402'] + + - repo: https://github.com/RobertCraigie/pyright-python + rev: v1.1.345 + hooks: + - id: pyright + + - repo: https://github.com/adrienverge/yamllint + rev: v1.35.1 + hooks: + - id: yamllint + args: [-d, relaxed] diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..65b27524 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,50 @@ +# CLAUDE.md + +This file provides guidance to Claude Code when working with the stack-orchestrator project. + +## Some rules to follow +NEVER speculate about the cause of something +NEVER assume your hypotheses are true without evidence + +ALWAYS clearly state when something is a hypothesis +ALWAYS use evidence from the systems your interacting with to support your claims and hypotheses + +## Key Principles + +### Development Guidelines +- **Single responsibility** - Each component has one clear purpose +- **Fail fast** - Let errors propagate, don't hide failures +- **DRY/KISS** - Minimize duplication and complexity + +## Development Philosophy: Conversational Literate Programming + +### Approach +This project follows principles inspired by literate programming, where development happens through explanatory conversation rather than code-first implementation. + +### Core Principles +- **Documentation-First**: All changes begin with discussion of intent and reasoning +- **Narrative-Driven**: Complex systems are explained through conversational exploration +- **Justification Required**: Every coding task must have a corresponding TODO.md item explaining the "why" +- **Iterative Understanding**: Architecture and implementation evolve through dialogue + +### Working Method +1. **Explore and Understand**: Read existing code to understand current state +2. **Discuss Architecture**: Workshop complex design decisions through conversation +3. **Document Intent**: Update TODO.md with clear justification before coding +4. **Explain Changes**: Each modification includes reasoning and context +5. **Maintain Narrative**: Conversations serve as living documentation of design evolution + +### Implementation Guidelines +- Treat conversations as primary documentation +- Explain architectural decisions before implementing +- Use TODO.md as the "literate document" that justifies all work +- Maintain clear narrative threads across sessions +- Workshop complex ideas before coding + +This approach treats the human-AI collaboration as a form of **conversational literate programming** where understanding emerges through dialogue before code implementation. + +## Insights and Observations + +### Design Principles +- **When something times out that doesn't mean it needs a longer timeout it means something that was expected never happened, not that we need to wait longer for it.** +- **NEVER change a timeout because you believe something truncated, you don't understand timeouts, don't edit them unless told to explicitly by user.** diff --git a/LICENSE b/LICENSE index bd018522..331f7cfa 100644 --- a/LICENSE +++ b/LICENSE @@ -658,4 +658,4 @@ You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see - . \ No newline at end of file + . diff --git a/README.md b/README.md index ef504295..375491bf 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ curl -SL https://github.com/docker/compose/releases/download/v2.11.2/docker-comp chmod +x ~/.docker/cli-plugins/docker-compose ``` -Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be +Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory. Now, having selected that directory, download the latest release from [this page](https://git.vdb.to/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable: @@ -78,5 +78,3 @@ See the [CONTRIBUTING.md](/docs/CONTRIBUTING.md) for developer mode install. ## Platform Support Native aarm64 is _not_ currently supported. x64 emulation on ARM64 macos should work (not yet tested). - - diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..349530c8 --- /dev/null +++ b/TODO.md @@ -0,0 +1,16 @@ +# TODO + +## Features Needed + +### Update Stack Command +We need an "update stack" command in stack orchestrator and cleaner documentation regarding how to do continuous deployment with and without payments. + +**Context**: Currently, `deploy init` generates a spec file and `deploy create` creates a deployment directory. The `deployment update` command (added by Thomas Lackey) only syncs env vars and restarts - it doesn't regenerate configurations. There's a gap in the workflow for updating stack configurations after initial deployment. + +## Architecture Refactoring + +### Separate Deployer from Stack Orchestrator CLI +The deployer logic should be decoupled from the CLI tool to allow independent development and reuse. + +### Separate Stacks from Stack Orchestrator Repo +Stacks should live in their own repositories, not bundled with the orchestrator tool. This allows stacks to evolve independently and be maintained by different teams. diff --git a/docs/fetching-containers.md b/docs/fetching-containers.md index 50a8b5ff..3e3d621c 100644 --- a/docs/fetching-containers.md +++ b/docs/fetching-containers.md @@ -1,9 +1,9 @@ # Fetching pre-built container images -When Stack Orchestrator deploys a stack containing a suite of one or more containers it expects images for those containers to be on the local machine with a tag of the form `:local` Images for these containers can be built from source (and optionally base container images from public registries) with the `build-containers` subcommand. +When Stack Orchestrator deploys a stack containing a suite of one or more containers it expects images for those containers to be on the local machine with a tag of the form `:local` Images for these containers can be built from source (and optionally base container images from public registries) with the `build-containers` subcommand. However, the task of building a large number of containers from source may consume considerable time and machine resources. This is where the `fetch-containers` subcommand steps in. It is designed to work exactly like `build-containers` but instead the images, pre-built, are fetched from an image registry then re-tagged for deployment. It can be used in place of `build-containers` for any stack provided the necessary containers, built for the local machine architecture (e.g. arm64 or x86-64) have already been published in an image registry. ## Usage To use `fetch-containers`, provide an image registry path, a username and token/password with read access to the registry, and optionally specify `--force-local-overwrite`. If this argument is not specified, if there is already a locally built or previously fetched image for a stack container on the machine, it will not be overwritten and a warning issued. ``` $ laconic-so --stack mobymask-v3-demo fetch-containers --image-registry git.vdb.to/cerc-io --registry-username --registry-token --force-local-overwrite -``` \ No newline at end of file +``` diff --git a/docs/gitea-with-laconicd-fixturenet.md b/docs/gitea-with-laconicd-fixturenet.md index f9ed86ab..61391b41 100644 --- a/docs/gitea-with-laconicd-fixturenet.md +++ b/docs/gitea-with-laconicd-fixturenet.md @@ -7,7 +7,7 @@ Deploy a local Gitea server, publish NPM packages to it, then use those packages ```bash laconic-so --stack build-support build-containers laconic-so --stack package-registry setup-repositories -laconic-so --stack package-registry build-containers +laconic-so --stack package-registry build-containers laconic-so --stack package-registry deploy up ``` diff --git a/docs/helm-chart-generation.md b/docs/helm-chart-generation.md new file mode 100644 index 00000000..903ae2da --- /dev/null +++ b/docs/helm-chart-generation.md @@ -0,0 +1,113 @@ +# Helm Chart Generation + +Generate Kubernetes Helm charts from stack compose files using Kompose. + +## Prerequisites + +Install Kompose: + +```bash +# Linux +curl -L https://github.com/kubernetes/kompose/releases/download/v1.34.0/kompose-linux-amd64 -o kompose +chmod +x kompose +sudo mv kompose /usr/local/bin/ + +# macOS +brew install kompose + +# Verify +kompose version +``` + +## Usage + +### 1. Create spec file + +```bash +laconic-so --stack deploy --deploy-to k8s init \ + --kube-config ~/.kube/config \ + --output spec.yml +``` + +### 2. Generate Helm chart + +```bash +laconic-so --stack deploy create \ + --spec-file spec.yml \ + --deployment-dir my-deployment \ + --helm-chart +``` + +### 3. Deploy to Kubernetes + +```bash +helm install my-release my-deployment/chart +kubectl get pods -n zenith +``` + +## Output Structure + +```bash +my-deployment/ +├── spec.yml # Reference +├── stack.yml # Reference +└── chart/ # Helm chart + ├── Chart.yaml + ├── README.md + └── templates/ + └── *.yaml +``` + +## Example + +```bash +# Generate chart for stage1-zenithd +laconic-so --stack stage1-zenithd deploy --deploy-to k8s init \ + --kube-config ~/.kube/config \ + --output stage1-spec.yml + +laconic-so --stack stage1-zenithd deploy create \ + --spec-file stage1-spec.yml \ + --deployment-dir stage1-deployment \ + --helm-chart + +# Deploy +helm install stage1-zenithd stage1-deployment/chart +``` + +## Production Deployment (TODO) + +### Local Development + +```bash +# Access services using port-forward +kubectl port-forward service/zenithd 26657:26657 +kubectl port-forward service/nginx-api-proxy 1317:80 +kubectl port-forward service/cosmos-explorer 4173:4173 +``` + +### Production Access Options + +- Option 1: Ingress + cert-manager (Recommended) + - Install ingress-nginx + cert-manager + - Point DNS to cluster LoadBalancer IP + - Auto-provisions Let's Encrypt TLS certs + - Access: `https://api.zenith.example.com` +- Option 2: Cloud LoadBalancer + - Use cloud provider's LoadBalancer service type + - Point DNS to assigned external IP + - Manual TLS cert management +- Option 3: Bare Metal (MetalLB + Ingress) + - MetalLB provides LoadBalancer IPs from local network + - Same Ingress setup as cloud +- Option 4: NodePort + External Proxy + - Expose services on 30000-32767 range + - External nginx/Caddy proxies 80/443 → NodePort + - Manual cert management + +### Changes Needed + +- Add Ingress template to charts +- Add TLS configuration to values.yaml +- Document cert-manager setup +- Add production deployment guide diff --git a/docs/k8s-deployment-enhancements.md b/docs/k8s-deployment-enhancements.md index 424d529f..d7838058 100644 --- a/docs/k8s-deployment-enhancements.md +++ b/docs/k8s-deployment-enhancements.md @@ -24,4 +24,3 @@ node-tolerations: value: typeb ``` This example denotes that the stack's pods will tolerate a taint: `nodetype=typeb` - diff --git a/docs/release-process.md b/docs/release-process.md index c1c6893f..abe51922 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -26,4 +26,3 @@ $ ./scripts/tag_new_release.sh 1 0 17 $ ./scripts/build_shiv_package.sh $ ./scripts/publish_shiv_package_github.sh 1 0 17 ``` - diff --git a/docs/spec.md b/docs/spec.md index aa09274c..863cfb81 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -4,9 +4,9 @@ Note: this page is out of date (but still useful) - it will no longer be useful ## Implementation -The orchestrator's operation is driven by files shown below. +The orchestrator's operation is driven by files shown below. -- `repository-list.txt` contains the list of git repositories; +- `repository-list.txt` contains the list of git repositories; - `container-image-list.txt` contains the list of container image names - `pod-list.txt` specifies the set of compose components (corresponding to individual docker-compose-xxx.yml files which may in turn specify more than one container). - `container-build/` contains the files required to build each container image diff --git a/docs/webapp.md b/docs/webapp.md index fcf4ffcb..fdd9c64a 100644 --- a/docs/webapp.md +++ b/docs/webapp.md @@ -7,7 +7,7 @@ compilation and static page generation are separated in the `build-webapp` and ` This offers much more flexibilty than standard Next.js build methods, since any environment variables accessed via `process.env`, whether for pages or for API, will have values drawn from their runtime deployment environment, -not their build environment. +not their build environment. ## Building diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..638d4ce8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,110 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "laconic-stack-orchestrator" +version = "1.1.0" +description = "Orchestrates deployment of the Laconic stack" +readme = "README.md" +license = {text = "GNU Affero General Public License"} +authors = [ + {name = "Cerc", email = "info@cerc.io"} +] +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: Python :: 3.8", + "Operating System :: OS Independent", +] +dependencies = [ + "python-decouple>=3.8", + "python-dotenv==1.0.0", + "GitPython>=3.1.32", + "tqdm>=4.65.0", + "python-on-whales>=0.64.0", + "click>=8.1.6", + "PyYAML>=6.0.1", + "ruamel.yaml>=0.17.32", + "pydantic==1.10.9", + "tomli==2.0.1", + "validators==0.22.0", + "kubernetes>=28.1.0", + "humanfriendly>=10.0", + "python-gnupg>=0.5.2", + "requests>=2.3.2", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=22.0.0", + "flake8>=5.0.0", + "pyright>=1.1.0", + "yamllint>=1.28.0", + "pre-commit>=3.0.0", +] + +[project.scripts] +laconic-so = "stack_orchestrator.main:cli" + +[project.urls] +Homepage = "https://git.vdb.to/cerc-io/stack-orchestrator" + +[tool.setuptools.packages.find] +where = ["."] + +[tool.setuptools.package-data] +"*" = ["data/**"] + +[tool.black] +line-length = 88 +target-version = ['py38'] + +[tool.flake8] +max-line-length = 88 +extend-ignore = ["E203", "W503", "E402"] + +[tool.pyright] +pythonVersion = "3.9" +typeCheckingMode = "basic" +reportMissingImports = "none" +reportMissingModuleSource = "none" +reportUnusedImport = "error" +include = ["stack_orchestrator/**/*.py", "tests/**/*.py"] +exclude = ["**/build/**", "**/__pycache__/**"] + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "e2e: marks tests as end-to-end (requires real infrastructure)", +] +addopts = [ + "--cov", + "--cov-report=term-missing", + "--cov-report=html", + "--strict-markers", +] +asyncio_default_fixture_loop_scope = "function" + +[tool.coverage.run] +source = ["stack_orchestrator"] +disable_warnings = ["couldnt-parse"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", +] diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 00000000..3675c660 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,9 @@ +{ + "pythonVersion": "3.9", + "typeCheckingMode": "basic", + "reportMissingImports": "none", + "reportMissingModuleSource": "none", + "reportUnusedImport": "error", + "include": ["stack_orchestrator/**/*.py", "tests/**/*.py"], + "exclude": ["**/build/**", "**/__pycache__/**"] +} diff --git a/scripts/publish_shiv_package_github.sh b/scripts/publish_shiv_package_github.sh index 72ce8959..3912581b 100755 --- a/scripts/publish_shiv_package_github.sh +++ b/scripts/publish_shiv_package_github.sh @@ -4,7 +4,7 @@ # https://github.com/cerc-io/github-release-api # User must define: CERC_GH_RELEASE_SCRIPTS_DIR # pointing to the location of that cloned repository -# e.g. +# e.g. # cd ~/projects # git clone https://github.com/cerc-io/github-release-api # cd ./stack-orchestrator diff --git a/scripts/quick-install-linux.sh b/scripts/quick-install-linux.sh index b8642416..ea251c6b 100755 --- a/scripts/quick-install-linux.sh +++ b/scripts/quick-install-linux.sh @@ -94,7 +94,7 @@ sudo apt -y install jq # laconic-so depends on git sudo apt -y install git # curl used below -sudo apt -y install curl +sudo apt -y install curl # docker repo add depends on gnupg and updated ca-certificates sudo apt -y install ca-certificates gnupg diff --git a/scripts/tag_new_release.sh b/scripts/tag_new_release.sh index 193b5d8a..99e91f87 100755 --- a/scripts/tag_new_release.sh +++ b/scripts/tag_new_release.sh @@ -3,7 +3,7 @@ # Uses this script package to tag a new release: # User must define: CERC_GH_RELEASE_SCRIPTS_DIR # pointing to the location of that cloned repository -# e.g. +# e.g. # cd ~/projects # git clone https://github.com/cerc-io/github-release-api # cd ./stack-orchestrator diff --git a/setup.py b/setup.py index ace0d536..b295802f 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,7 @@ -# See https://medium.com/nerd-for-tech/how-to-build-and-distribute-a-cli-tool-with-python-537ae41d9d78 +# See +# https://medium.com/nerd-for-tech/how-to-build-and-distribute-a-cli-tool-with-python-537ae41d9d78 from setuptools import setup, find_packages + with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() with open("requirements.txt", "r", encoding="utf-8") as fh: @@ -7,26 +9,26 @@ with open("requirements.txt", "r", encoding="utf-8") as fh: with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh: version = fh.readlines()[-1].strip(" \n") setup( - name='laconic-stack-orchestrator', + name="laconic-stack-orchestrator", version=version, - author='Cerc', - author_email='info@cerc.io', - license='GNU Affero General Public License', - description='Orchestrates deployment of the Laconic stack', + author="Cerc", + author_email="info@cerc.io", + license="GNU Affero General Public License", + description="Orchestrates deployment of the Laconic stack", long_description=long_description, long_description_content_type="text/markdown", - url='https://git.vdb.to/cerc-io/stack-orchestrator', - py_modules=['stack_orchestrator'], + url="https://git.vdb.to/cerc-io/stack-orchestrator", + py_modules=["stack_orchestrator"], packages=find_packages(), install_requires=[requirements], - python_requires='>=3.7', + python_requires=">=3.7", include_package_data=True, - package_data={'': ['data/**']}, + package_data={"": ["data/**"]}, classifiers=[ "Programming Language :: Python :: 3.8", "Operating System :: OS Independent", ], entry_points={ - 'console_scripts': ['laconic-so=stack_orchestrator.main:cli'], - } + "console_scripts": ["laconic-so=stack_orchestrator.main:cli"], + }, ) diff --git a/stack_orchestrator/base.py b/stack_orchestrator/base.py index 811d085d..eb4b7e77 100644 --- a/stack_orchestrator/base.py +++ b/stack_orchestrator/base.py @@ -23,11 +23,10 @@ def get_stack(config, stack): if stack == "package-registry": return package_registry_stack(config, stack) else: - return base_stack(config, stack) + return default_stack(config, stack) class base_stack(ABC): - def __init__(self, config, stack): self.config = config self.stack = stack @@ -41,15 +40,27 @@ class base_stack(ABC): pass -class package_registry_stack(base_stack): +class default_stack(base_stack): + """Default stack implementation for stacks without specific handling.""" + def ensure_available(self): + return True + + def get_url(self): + return None + + +class package_registry_stack(base_stack): def ensure_available(self): self.url = "" # Check if we were given an external registry URL url_from_environment = os.environ.get("CERC_NPM_REGISTRY_URL") if url_from_environment: if self.config.verbose: - print(f"Using package registry url from CERC_NPM_REGISTRY_URL: {url_from_environment}") + print( + f"Using package registry url from CERC_NPM_REGISTRY_URL: " + f"{url_from_environment}" + ) self.url = url_from_environment else: # Otherwise we expect to use the local package-registry stack @@ -62,10 +73,16 @@ class package_registry_stack(base_stack): # TODO: get url from deploy-stack self.url = "http://gitea.local:3000/api/packages/cerc-io/npm/" else: - # If not, print a message about how to start it and return fail to the caller - print("ERROR: The package-registry stack is not running, and no external registry " - "specified with CERC_NPM_REGISTRY_URL") - print("ERROR: Start the local package registry with: laconic-so --stack package-registry deploy-system up") + # If not, print a message about how to start it and return fail to the + # caller + print( + "ERROR: The package-registry stack is not running, " + "and no external registry specified with CERC_NPM_REGISTRY_URL" + ) + print( + "ERROR: Start the local package registry with: " + "laconic-so --stack package-registry deploy-system up" + ) return False return True @@ -76,7 +93,9 @@ class package_registry_stack(base_stack): def get_npm_registry_url(): # If an auth token is not defined, we assume the default should be the cerc registry # If an auth token is defined, we assume the local gitea should be used. - default_npm_registry_url = "http://gitea.local:3000/api/packages/cerc-io/npm/" if config( - "CERC_NPM_AUTH_TOKEN", default=None - ) else "https://git.vdb.to/api/packages/cerc-io/npm/" + default_npm_registry_url = ( + "http://gitea.local:3000/api/packages/cerc-io/npm/" + if config("CERC_NPM_AUTH_TOKEN", default=None) + else "https://git.vdb.to/api/packages/cerc-io/npm/" + ) return config("CERC_NPM_REGISTRY_URL", default=default_npm_registry_url) diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index 2b78306b..4717b7a6 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -18,7 +18,8 @@ # env vars: # CERC_REPO_BASE_DIR defaults to ~/cerc -# TODO: display the available list of containers; allow re-build of either all or specific containers +# TODO: display the available list of containers; +# allow re-build of either all or specific containers import os import sys @@ -34,14 +35,17 @@ from stack_orchestrator.build.publish import publish_image from stack_orchestrator.build.build_util import get_containers_in_scope # TODO: find a place for this -# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" +# epilog="Config provided either in .env or settings.ini or env vars: +# CERC_REPO_BASE_DIR (defaults to ~/cerc)" -def make_container_build_env(dev_root_path: str, - container_build_dir: str, - debug: bool, - force_rebuild: bool, - extra_build_args: str): +def make_container_build_env( + dev_root_path: str, + container_build_dir: str, + debug: bool, + force_rebuild: bool, + extra_build_args: str, +): container_build_env = { "CERC_NPM_REGISTRY_URL": get_npm_registry_url(), "CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""), @@ -50,11 +54,15 @@ def make_container_build_env(dev_root_path: str, "CERC_CONTAINER_BASE_DIR": container_build_dir, "CERC_HOST_UID": f"{os.getuid()}", "CERC_HOST_GID": f"{os.getgid()}", - "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0") + "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0"), } container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {}) container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {}) - container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {}) + container_build_env.update( + {"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} + if extra_build_args + else {} + ) docker_host_env = os.getenv("DOCKER_HOST") if docker_host_env: container_build_env.update({"DOCKER_HOST": docker_host_env}) @@ -67,12 +75,18 @@ def process_container(build_context: BuildContext) -> bool: print(f"Building: {build_context.container}") default_container_tag = f"{build_context.container}:local" - build_context.container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag}) + build_context.container_build_env.update( + {"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag} + ) # Check if this is in an external stack if stack_is_external(build_context.stack): - container_parent_dir = Path(build_context.stack).parent.parent.joinpath("container-build") - temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-")) + container_parent_dir = Path(build_context.stack).parent.parent.joinpath( + "container-build" + ) + temp_build_dir = container_parent_dir.joinpath( + build_context.container.replace("/", "-") + ) temp_build_script_filename = temp_build_dir.joinpath("build.sh") # Now check if the container exists in the external stack. if not temp_build_script_filename.exists(): @@ -90,21 +104,34 @@ def process_container(build_context: BuildContext) -> bool: build_command = build_script_filename.as_posix() else: if opts.o.verbose: - print(f"No script file found: {build_script_filename}, using default build script") - repo_dir = build_context.container.split('/')[1] - # TODO: make this less of a hack -- should be specified in some metadata somewhere - # Check if we have a repo for this container. If not, set the context dir to the container-build subdir + print( + f"No script file found: {build_script_filename}, " + "using default build script" + ) + repo_dir = build_context.container.split("/")[1] + # TODO: make this less of a hack -- should be specified in + # some metadata somewhere. Check if we have a repo for this + # container. If not, set the context dir to container-build subdir repo_full_path = os.path.join(build_context.dev_root_path, repo_dir) - repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir - build_command = os.path.join(build_context.container_build_dir, - "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}" + repo_dir_or_build_dir = ( + repo_full_path if os.path.exists(repo_full_path) else build_dir + ) + build_command = ( + os.path.join(build_context.container_build_dir, "default-build.sh") + + f" {default_container_tag} {repo_dir_or_build_dir}" + ) if not opts.o.dry_run: # No PATH at all causes failures with podman. if "PATH" not in build_context.container_build_env: build_context.container_build_env["PATH"] = os.environ["PATH"] if opts.o.verbose: - print(f"Executing: {build_command} with environment: {build_context.container_build_env}") - build_result = subprocess.run(build_command, shell=True, env=build_context.container_build_env) + print( + f"Executing: {build_command} with environment: " + f"{build_context.container_build_env}" + ) + build_result = subprocess.run( + build_command, shell=True, env=build_context.container_build_env + ) if opts.o.verbose: print(f"Return code is: {build_result.returncode}") if build_result.returncode != 0: @@ -117,33 +144,61 @@ def process_container(build_context: BuildContext) -> bool: @click.command() -@click.option('--include', help="only build these containers") -@click.option('--exclude', help="don\'t build these containers") -@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild") +@click.option("--include", help="only build these containers") +@click.option("--exclude", help="don't build these containers") +@click.option( + "--force-rebuild", + is_flag=True, + default=False, + help="Override dependency checking -- always rebuild", +) @click.option("--extra-build-args", help="Supply extra arguments to build") -@click.option("--publish-images", is_flag=True, default=False, help="Publish the built images in the specified image registry") -@click.option("--image-registry", help="Specify the image registry for --publish-images") +@click.option( + "--publish-images", + is_flag=True, + default=False, + help="Publish the built images in the specified image registry", +) +@click.option( + "--image-registry", help="Specify the image registry for --publish-images" +) @click.pass_context -def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_images, image_registry): - '''build the set of containers required for a complete stack''' +def command( + ctx, + include, + exclude, + force_rebuild, + extra_build_args, + publish_images, + image_registry, +): + """build the set of containers required for a complete stack""" local_stack = ctx.obj.local_stack stack = ctx.obj.stack - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build") + # See: https://stackoverflow.com/questions/25389095/ + # python-get-path-of-root-project-structure + container_build_dir = ( + Path(__file__).absolute().parent.parent.joinpath("data", "container-build") + ) if local_stack: - dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] - print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")] + print( + f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " + f"{dev_root_path}" + ) else: - dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + dev_root_path = os.path.expanduser( + config("CERC_REPO_BASE_DIR", default="~/cerc") + ) if not opts.o.quiet: - print(f'Dev Root is: {dev_root_path}') + print(f"Dev Root is: {dev_root_path}") if not os.path.isdir(dev_root_path): - print('Dev root directory doesn\'t exist, creating') + print("Dev root directory doesn't exist, creating") if publish_images: if not image_registry: @@ -151,21 +206,22 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag containers_in_scope = get_containers_in_scope(stack) - container_build_env = make_container_build_env(dev_root_path, - container_build_dir, - opts.o.debug, - force_rebuild, - extra_build_args) + container_build_env = make_container_build_env( + dev_root_path, + container_build_dir, + opts.o.debug, + force_rebuild, + extra_build_args, + ) for container in containers_in_scope: if include_exclude_check(container, include, exclude): - build_context = BuildContext( stack, container, container_build_dir, container_build_env, - dev_root_path + dev_root_path, ) result = process_container(build_context) if result: @@ -174,10 +230,16 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag else: print(f"Error running build for {build_context.container}") if not opts.o.continue_on_error: - error_exit("container build failed and --continue-on-error not set, exiting") + error_exit( + "container build failed and --continue-on-error " + "not set, exiting" + ) sys.exit(1) else: - print("****** Container Build Error, continuing because --continue-on-error is set") + print( + "****** Container Build Error, continuing because " + "--continue-on-error is set" + ) else: if opts.o.verbose: print(f"Excluding: {container}") diff --git a/stack_orchestrator/build/build_npms.py b/stack_orchestrator/build/build_npms.py index c8e3af43..00992546 100644 --- a/stack_orchestrator/build/build_npms.py +++ b/stack_orchestrator/build/build_npms.py @@ -32,14 +32,18 @@ builder_js_image_name = "cerc/builder-js:local" @click.command() -@click.option('--include', help="only build these packages") -@click.option('--exclude', help="don\'t build these packages") -@click.option("--force-rebuild", is_flag=True, default=False, - help="Override existing target package version check -- force rebuild") +@click.option("--include", help="only build these packages") +@click.option("--exclude", help="don't build these packages") +@click.option( + "--force-rebuild", + is_flag=True, + default=False, + help="Override existing target package version check -- force rebuild", +) @click.option("--extra-build-args", help="Supply extra arguments to build") @click.pass_context def command(ctx, include, exclude, force_rebuild, extra_build_args): - '''build the set of npm packages required for a complete stack''' + """build the set of npm packages required for a complete stack""" quiet = ctx.obj.quiet verbose = ctx.obj.verbose @@ -65,45 +69,54 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): sys.exit(1) if local_stack: - dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] - print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")] + print( + f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " + f"{dev_root_path}" + ) else: - dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + dev_root_path = os.path.expanduser( + config("CERC_REPO_BASE_DIR", default="~/cerc") + ) build_root_path = os.path.join(dev_root_path, "build-trees") if verbose: - print(f'Dev Root is: {dev_root_path}') + print(f"Dev Root is: {dev_root_path}") if not os.path.isdir(dev_root_path): - print('Dev root directory doesn\'t exist, creating') + print("Dev root directory doesn't exist, creating") os.makedirs(dev_root_path) if not os.path.isdir(dev_root_path): - print('Build root directory doesn\'t exist, creating') + print("Build root directory doesn't exist, creating") os.makedirs(build_root_path) # See: https://stackoverflow.com/a/20885799/1701505 from stack_orchestrator import data - with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file: + + with importlib.resources.open_text( + data, "npm-package-list.txt" + ) as package_list_file: all_packages = package_list_file.read().splitlines() packages_in_scope = [] if stack: stack_config = get_parsed_stack_config(stack) # TODO: syntax check the input here - packages_in_scope = stack_config['npms'] + packages_in_scope = stack_config["npms"] else: packages_in_scope = all_packages if verbose: - print(f'Packages: {packages_in_scope}') + print(f"Packages: {packages_in_scope}") def build_package(package): if not quiet: print(f"Building npm package: {package}") repo_dir = package repo_full_path = os.path.join(dev_root_path, repo_dir) - # Copy the repo and build that to avoid propagating JS tooling file changes back into the cloned repo + # Copy the repo and build that to avoid propagating + # JS tooling file changes back into the cloned repo repo_copy_path = os.path.join(build_root_path, repo_dir) # First delete any old build tree if os.path.isdir(repo_copy_path): @@ -116,41 +129,63 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): print(f"Copying build tree from: {repo_full_path} to: {repo_copy_path}") if not dry_run: copytree(repo_full_path, repo_copy_path) - build_command = ["sh", "-c", f"cd /workspace && build-npm-package-local-dependencies.sh {npm_registry_url}"] + build_command = [ + "sh", + "-c", + "cd /workspace && " + f"build-npm-package-local-dependencies.sh {npm_registry_url}", + ] if not dry_run: if verbose: print(f"Executing: {build_command}") # Originally we used the PEP 584 merge operator: - # envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {}) - # but that isn't available in Python 3.8 (default in Ubuntu 20) so for now we use dict.update: - envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token, - "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml" # Convention used by our web app packages - } + # envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | + # ({"CERC_SCRIPT_DEBUG": "true"} if debug else {}) + # but that isn't available in Python 3.8 (default in Ubuntu 20) + # so for now we use dict.update: + envs = { + "CERC_NPM_AUTH_TOKEN": npm_registry_url_token, + # Convention used by our web app packages + "LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml", + } envs.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {}) envs.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {}) - envs.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {}) + envs.update( + {"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} + if extra_build_args + else {} + ) try: - docker.run(builder_js_image_name, - remove=True, - interactive=True, - tty=True, - user=f"{os.getuid()}:{os.getgid()}", - envs=envs, - # TODO: detect this host name in npm_registry_url rather than hard-wiring it - add_hosts=[("gitea.local", "host-gateway")], - volumes=[(repo_copy_path, "/workspace")], - command=build_command - ) - # Note that although the docs say that build_result should contain - # the command output as a string, in reality it is always the empty string. - # Since we detect errors via catching exceptions below, we can safely ignore it here. + docker.run( + builder_js_image_name, + remove=True, + interactive=True, + tty=True, + user=f"{os.getuid()}:{os.getgid()}", + envs=envs, + # TODO: detect this host name in npm_registry_url + # rather than hard-wiring it + add_hosts=[("gitea.local", "host-gateway")], + volumes=[(repo_copy_path, "/workspace")], + command=build_command, + ) + # Note that although the docs say that build_result should + # contain the command output as a string, in reality it is + # always the empty string. Since we detect errors via catching + # exceptions below, we can safely ignore it here. except DockerException as e: print(f"Error executing build for {package} in container:\n {e}") if not continue_on_error: - print("FATAL Error: build failed and --continue-on-error not set, exiting") + print( + "FATAL Error: build failed and --continue-on-error " + "not set, exiting" + ) sys.exit(1) else: - print("****** Build Error, continuing because --continue-on-error is set") + print( + "****** Build Error, continuing because " + "--continue-on-error is set" + ) else: print("Skipped") @@ -168,6 +203,12 @@ def _ensure_prerequisites(): # Tell the user how to build it if not images = docker.image.list(builder_js_image_name) if len(images) == 0: - print(f"FATAL: builder image: {builder_js_image_name} is required but was not found") - print("Please run this command to create it: laconic-so --stack build-support build-containers") + print( + f"FATAL: builder image: {builder_js_image_name} is required " + "but was not found" + ) + print( + "Please run this command to create it: " + "laconic-so --stack build-support build-containers" + ) sys.exit(1) diff --git a/stack_orchestrator/build/build_types.py b/stack_orchestrator/build/build_types.py index 971188fe..53b24932 100644 --- a/stack_orchestrator/build/build_types.py +++ b/stack_orchestrator/build/build_types.py @@ -24,6 +24,5 @@ class BuildContext: stack: str container: str container_build_dir: Path - container_build_env: Mapping[str,str] + container_build_env: Mapping[str, str] dev_root_path: str - diff --git a/stack_orchestrator/build/build_util.py b/stack_orchestrator/build/build_util.py index 15be1f9b..a8a0c395 100644 --- a/stack_orchestrator/build/build_util.py +++ b/stack_orchestrator/build/build_util.py @@ -20,21 +20,23 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit def get_containers_in_scope(stack: str): - containers_in_scope = [] if stack: stack_config = get_parsed_stack_config(stack) if "containers" not in stack_config or stack_config["containers"] is None: warn_exit(f"stack {stack} does not define any containers") - containers_in_scope = stack_config['containers'] + containers_in_scope = stack_config["containers"] else: # See: https://stackoverflow.com/a/20885799/1701505 from stack_orchestrator import data - with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: + + with importlib.resources.open_text( + data, "container-image-list.txt" + ) as container_list_file: containers_in_scope = container_list_file.read().splitlines() if opts.o.verbose: - print(f'Containers: {containers_in_scope}') + print(f"Containers: {containers_in_scope}") if stack: print(f"Stack: {stack}") diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index 1021f4bf..f204df82 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -18,7 +18,8 @@ # env vars: # CERC_REPO_BASE_DIR defaults to ~/cerc -# TODO: display the available list of containers; allow re-build of either all or specific containers +# TODO: display the available list of containers; +# allow re-build of either all or specific containers import os import sys @@ -32,40 +33,55 @@ from stack_orchestrator.build.build_types import BuildContext @click.command() -@click.option('--base-container') -@click.option('--source-repo', help="directory containing the webapp to build", required=True) -@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild") +@click.option("--base-container") +@click.option( + "--source-repo", help="directory containing the webapp to build", required=True +) +@click.option( + "--force-rebuild", + is_flag=True, + default=False, + help="Override dependency checking -- always rebuild", +) @click.option("--extra-build-args", help="Supply extra arguments to build") @click.option("--tag", help="Container tag (default: cerc/:local)") @click.pass_context def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag): - '''build the specified webapp container''' + """build the specified webapp container""" logger = TimedLogger() - quiet = ctx.obj.quiet debug = ctx.obj.debug verbose = ctx.obj.verbose local_stack = ctx.obj.local_stack stack = ctx.obj.stack - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build") + # See: https://stackoverflow.com/questions/25389095/ + # python-get-path-of-root-project-structure + container_build_dir = ( + Path(__file__).absolute().parent.parent.joinpath("data", "container-build") + ) if local_stack: - dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] - logger.log(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")] + logger.log( + f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " + f"{dev_root_path}" + ) else: - dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + dev_root_path = os.path.expanduser( + config("CERC_REPO_BASE_DIR", default="~/cerc") + ) if verbose: - logger.log(f'Dev Root is: {dev_root_path}') + logger.log(f"Dev Root is: {dev_root_path}") if not base_container: base_container = determine_base_container(source_repo) # First build the base container. - container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug, - force_rebuild, extra_build_args) + container_build_env = build_containers.make_container_build_env( + dev_root_path, container_build_dir, debug, force_rebuild, extra_build_args + ) if verbose: logger.log(f"Building base container: {base_container}") @@ -85,12 +101,13 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t if verbose: logger.log(f"Base container {base_container} build finished.") - # Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir. + # Now build the target webapp. We use the same build script, + # but with a different Dockerfile and work dir. container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true" container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo) - container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir, - base_container.replace("/", "-"), - "Dockerfile.webapp") + container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join( + container_build_dir, base_container.replace("/", "-"), "Dockerfile.webapp" + ) if not tag: webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] tag = f"cerc/{webapp_name}:local" diff --git a/stack_orchestrator/build/fetch_containers.py b/stack_orchestrator/build/fetch_containers.py index ed7d3675..e0f31dd0 100644 --- a/stack_orchestrator/build/fetch_containers.py +++ b/stack_orchestrator/build/fetch_containers.py @@ -52,7 +52,8 @@ def _local_tag_for(container: str): # See: https://docker-docs.uclv.cu/registry/spec/api/ # Emulate this: -# $ curl -u "my-username:my-token" -X GET "https:///v2/cerc-io/cerc/test-container/tags/list" +# $ curl -u "my-username:my-token" -X GET \ +# "https:///v2/cerc-io/cerc/test-container/tags/list" # {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]} def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]: # registry looks like: git.vdb.to/cerc-io @@ -60,7 +61,9 @@ def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list" if opts.o.debug: print(f"Fetching tags from: {url}") - response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token)) + response = requests.get( + url, auth=(registry_info.registry_username, registry_info.registry_token) + ) if response.status_code == 200: tag_info = response.json() if opts.o.debug: @@ -68,7 +71,10 @@ def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List tags_array = tag_info["tags"] return tags_array else: - error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}") + error_exit( + f"failed to fetch tags from image registry, " + f"status code: {response.status_code}" + ) def _find_latest(candidate_tags: List[str]): @@ -79,9 +85,9 @@ def _find_latest(candidate_tags: List[str]): return sorted_candidates[-1] -def _filter_for_platform(container: str, - registry_info: RegistryInfo, - tag_list: List[str]) -> List[str] : +def _filter_for_platform( + container: str, registry_info: RegistryInfo, tag_list: List[str] +) -> List[str]: filtered_tags = [] this_machine = platform.machine() # Translate between Python and docker platform names @@ -98,7 +104,7 @@ def _filter_for_platform(container: str, manifest = manifest_cmd.inspect_verbose(remote_tag) if opts.o.debug: print(f"manifest: {manifest}") - image_architecture = manifest["Descriptor"]["platform"]["architecture"] + image_architecture = manifest["Descriptor"]["platform"]["architecture"] if opts.o.debug: print(f"image_architecture: {image_architecture}") if this_machine == image_architecture: @@ -137,21 +143,44 @@ def _add_local_tag(remote_tag: str, registry: str, local_tag: str): @click.command() -@click.option('--include', help="only fetch these containers") -@click.option('--exclude', help="don\'t fetch these containers") -@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present") -@click.option("--image-registry", required=True, help="Specify the image registry to fetch from") -@click.option("--registry-username", required=True, help="Specify the image registry username") -@click.option("--registry-token", required=True, help="Specify the image registry access token") +@click.option("--include", help="only fetch these containers") +@click.option("--exclude", help="don't fetch these containers") +@click.option( + "--force-local-overwrite", + is_flag=True, + default=False, + help="Overwrite a locally built image, if present", +) +@click.option( + "--image-registry", required=True, help="Specify the image registry to fetch from" +) +@click.option( + "--registry-username", required=True, help="Specify the image registry username" +) +@click.option( + "--registry-token", required=True, help="Specify the image registry access token" +) @click.pass_context -def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token): - '''EXPERIMENTAL: fetch the images for a stack from remote registry''' +def command( + ctx, + include, + exclude, + force_local_overwrite, + image_registry, + registry_username, + registry_token, +): + """EXPERIMENTAL: fetch the images for a stack from remote registry""" registry_info = RegistryInfo(image_registry, registry_username, registry_token) docker = DockerClient() if not opts.o.quiet: print("Logging into container registry:") - docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token) + docker.login( + registry_info.registry, + registry_info.registry_username, + registry_info.registry_token, + ) # Generate list of target containers stack = ctx.obj.stack containers_in_scope = get_containers_in_scope(stack) @@ -172,19 +201,24 @@ def command(ctx, include, exclude, force_local_overwrite, image_registry, regist print(f"Fetching: {image_to_fetch}") _fetch_image(image_to_fetch, registry_info) # Now check if the target container already exists exists locally already - if (_exists_locally(container)): + if _exists_locally(container): if not opts.o.quiet: print(f"Container image {container} already exists locally") # if so, fail unless the user specified force-local-overwrite - if (force_local_overwrite): + if force_local_overwrite: # In that case remove the existing :local tag if not opts.o.quiet: - print(f"Warning: overwriting local tag from this image: {container} because " - "--force-local-overwrite was specified") + print( + f"Warning: overwriting local tag from this image: " + f"{container} because --force-local-overwrite was specified" + ) else: if not opts.o.quiet: - print(f"Skipping local tagging for this image: {container} because that would " - "overwrite an existing :local tagged image, use --force-local-overwrite to do so.") + print( + f"Skipping local tagging for this image: {container} " + "because that would overwrite an existing :local tagged " + "image, use --force-local-overwrite to do so." + ) continue # Tag the fetched image with the :local tag _add_local_tag(image_to_fetch, image_registry, local_tag) @@ -192,4 +226,7 @@ def command(ctx, include, exclude, force_local_overwrite, image_registry, regist if opts.o.verbose: print(f"Excluding: {container}") if not all_containers_found: - print("Warning: couldn't find usable images for one or more containers, this stack will not deploy") + print( + "Warning: couldn't find usable images for one or more containers, " + "this stack will not deploy" + ) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index 07fc68f4..49dfa193 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -39,3 +39,8 @@ node_affinities_key = "node-affinities" node_tolerations_key = "node-tolerations" kind_config_filename = "kind-config.yml" kube_config_filename = "kubeconfig.yml" +cri_base_filename = "cri-base.json" +unlimited_memlock_key = "unlimited-memlock" +runtime_class_key = "runtime-class" +high_memlock_runtime = "high-memlock" +high_memlock_spec_filename = "high-memlock-spec.json" diff --git a/stack_orchestrator/data/compose/docker-compose-fixturenet-blast.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-blast.yml index e6373796..679013ca 100644 --- a/stack_orchestrator/data/compose/docker-compose-fixturenet-blast.yml +++ b/stack_orchestrator/data/compose/docker-compose-fixturenet-blast.yml @@ -20,7 +20,7 @@ services: depends_on: generate-jwt: condition: service_completed_successfully - env_file: + env_file: - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config blast-geth: image: blastio/blast-geth:${NETWORK:-testnet-sepolia} @@ -51,7 +51,7 @@ services: --nodiscover --maxpeers=0 --rollup.disabletxpoolgossip=true - env_file: + env_file: - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config depends_on: geth-init: @@ -73,7 +73,7 @@ services: --rollup.config="/blast/rollup.json" depends_on: - blast-geth - env_file: + env_file: - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config volumes: diff --git a/stack_orchestrator/data/compose/docker-compose-laconicd.yml b/stack_orchestrator/data/compose/docker-compose-laconicd.yml index 753283bd..772ab34f 100644 --- a/stack_orchestrator/data/compose/docker-compose-laconicd.yml +++ b/stack_orchestrator/data/compose/docker-compose-laconicd.yml @@ -14,4 +14,3 @@ services: - "9090" - "9091" - "1317" - diff --git a/stack_orchestrator/data/compose/docker-compose-mainnet-blast.yml b/stack_orchestrator/data/compose/docker-compose-mainnet-blast.yml index b8f69fab..2e61546f 100644 --- a/stack_orchestrator/data/compose/docker-compose-mainnet-blast.yml +++ b/stack_orchestrator/data/compose/docker-compose-mainnet-blast.yml @@ -19,7 +19,7 @@ services: depends_on: generate-jwt: condition: service_completed_successfully - env_file: + env_file: - ../config/mainnet-blast/${NETWORK:-mainnet}.config blast-geth: image: blastio/blast-geth:${NETWORK:-mainnet} @@ -53,7 +53,7 @@ services: --nodiscover --maxpeers=0 --rollup.disabletxpoolgossip=true - env_file: + env_file: - ../config/mainnet-blast/${NETWORK:-mainnet}.config depends_on: geth-init: @@ -76,7 +76,7 @@ services: --rollup.config="/blast/rollup.json" depends_on: - blast-geth - env_file: + env_file: - ../config/mainnet-blast/${NETWORK:-mainnet}.config volumes: diff --git a/stack_orchestrator/data/compose/docker-compose-mars.yml b/stack_orchestrator/data/compose/docker-compose-mars.yml index 193a90af..461b80b9 100644 --- a/stack_orchestrator/data/compose/docker-compose-mars.yml +++ b/stack_orchestrator/data/compose/docker-compose-mars.yml @@ -17,4 +17,3 @@ services: - URL_NEUTRON_TEST_REST=https://rest-palvus.pion-1.ntrn.tech - URL_NEUTRON_TEST_RPC=https://rpc-palvus.pion-1.ntrn.tech - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x - diff --git a/stack_orchestrator/data/compose/docker-compose-reth.yml b/stack_orchestrator/data/compose/docker-compose-reth.yml index a3973717..26836960 100644 --- a/stack_orchestrator/data/compose/docker-compose-reth.yml +++ b/stack_orchestrator/data/compose/docker-compose-reth.yml @@ -32,4 +32,4 @@ services: volumes: reth_data: lighthouse_data: - shared_data: \ No newline at end of file + shared_data: diff --git a/stack_orchestrator/data/compose/docker-compose-test-database.yml b/stack_orchestrator/data/compose/docker-compose-test-database.yml index 6b99cdab..4093382d 100644 --- a/stack_orchestrator/data/compose/docker-compose-test-database.yml +++ b/stack_orchestrator/data/compose/docker-compose-test-database.yml @@ -12,7 +12,7 @@ services: POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" ports: - "5432" - + test-client: image: cerc/test-database-client:local diff --git a/stack_orchestrator/data/config/fixturenet-blast/fixturenet.config b/stack_orchestrator/data/config/fixturenet-blast/fixturenet.config index bd891dbf..1082801c 100644 --- a/stack_orchestrator/data/config/fixturenet-blast/fixturenet.config +++ b/stack_orchestrator/data/config/fixturenet-blast/fixturenet.config @@ -1,2 +1,2 @@ GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.s2.testblast.io -OP_NODE_P2P_BOOTNODES=enr:-J-4QM3GLUFfKMSJQuP1UvuKQe8DyovE7Eaiit0l6By4zjTodkR4V8NWXJxNmlg8t8rP-Q-wp3jVmeAOml8cjMj__ROGAYznzb_HgmlkgnY0gmlwhA-cZ_eHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAiuDqvB-AsVSRmnnWr6OHfjgY8YfNclFy9p02flKzXnOg3RjcIJ2YYN1ZHCCdmE,enr:-J-4QDCVpByqQ8nFqCS9aHicqwUfXgzFDslvpEyYz19lvkHLIdtcIGp2d4q5dxHdjRNTO6HXCsnIKxUeuZSPcEbyVQCGAYznzz0RgmlkgnY0gmlwhANiQfuHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAy3AtF2Jh_aPdOohg506Hjmtx-fQ1AKmu71C7PfkWAw9g3RjcIJ2YYN1ZHCCdmE \ No newline at end of file +OP_NODE_P2P_BOOTNODES=enr:-J-4QM3GLUFfKMSJQuP1UvuKQe8DyovE7Eaiit0l6By4zjTodkR4V8NWXJxNmlg8t8rP-Q-wp3jVmeAOml8cjMj__ROGAYznzb_HgmlkgnY0gmlwhA-cZ_eHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAiuDqvB-AsVSRmnnWr6OHfjgY8YfNclFy9p02flKzXnOg3RjcIJ2YYN1ZHCCdmE,enr:-J-4QDCVpByqQ8nFqCS9aHicqwUfXgzFDslvpEyYz19lvkHLIdtcIGp2d4q5dxHdjRNTO6HXCsnIKxUeuZSPcEbyVQCGAYznzz0RgmlkgnY0gmlwhANiQfuHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAy3AtF2Jh_aPdOohg506Hjmtx-fQ1AKmu71C7PfkWAw9g3RjcIJ2YYN1ZHCCdmE diff --git a/stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json b/stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json index 21932a24..3ea31bf9 100644 --- a/stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json +++ b/stack_orchestrator/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json @@ -1411,4 +1411,4 @@ "uid": "nT9VeZoVk", "version": 2, "weekStart": "" -} \ No newline at end of file +} diff --git a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh index 23a2bc30..ce5e2b73 100755 --- a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh +++ b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh @@ -65,7 +65,7 @@ if [ -n "$CERC_L1_ADDRESS" ] && [ -n "$CERC_L1_PRIV_KEY" ]; then # Sequencer SEQ=$(echo "$wallet3" | awk '/Address:/{print $2}') SEQ_KEY=$(echo "$wallet3" | awk '/Private key:/{print $3}') - + echo "Funding accounts." wait_for_block 1 300 cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 5ether $PROPOSER --private-key $ADMIN_KEY diff --git a/stack_orchestrator/data/config/fixturenet-pocket/genesis.json b/stack_orchestrator/data/config/fixturenet-pocket/genesis.json index 4d92d5a6..eb0083c3 100644 --- a/stack_orchestrator/data/config/fixturenet-pocket/genesis.json +++ b/stack_orchestrator/data/config/fixturenet-pocket/genesis.json @@ -56,7 +56,7 @@ "value": "!validator-pubkey" } } - } + } ], "supply": [] }, @@ -269,4 +269,4 @@ "claims": null } } -} \ No newline at end of file +} diff --git a/stack_orchestrator/data/config/keycloak/import/cerc-realm.json b/stack_orchestrator/data/config/keycloak/import/cerc-realm.json index e1e9dc97..062da56b 100644 --- a/stack_orchestrator/data/config/keycloak/import/cerc-realm.json +++ b/stack_orchestrator/data/config/keycloak/import/cerc-realm.json @@ -2084,4 +2084,4 @@ "clientPolicies": { "policies": [] } -} \ No newline at end of file +} diff --git a/stack_orchestrator/data/config/mainnet-blast/import/cerc-realm.json b/stack_orchestrator/data/config/mainnet-blast/import/cerc-realm.json index b6b6b606..d7d82361 100644 --- a/stack_orchestrator/data/config/mainnet-blast/import/cerc-realm.json +++ b/stack_orchestrator/data/config/mainnet-blast/import/cerc-realm.json @@ -2388,4 +2388,4 @@ "clientPolicies": { "policies": [] } -} \ No newline at end of file +} diff --git a/stack_orchestrator/data/config/mainnet-blast/rollup.json b/stack_orchestrator/data/config/mainnet-blast/rollup.json index f59e8323..cd7ed05f 100644 --- a/stack_orchestrator/data/config/mainnet-blast/rollup.json +++ b/stack_orchestrator/data/config/mainnet-blast/rollup.json @@ -29,4 +29,3 @@ "l1_system_config_address": "0x5531dcff39ec1ec727c4c5d2fc49835368f805a9", "protocol_versions_address": "0x0000000000000000000000000000000000000000" } - \ No newline at end of file diff --git a/stack_orchestrator/data/config/mainnet-eth-keycloak/import/cerc-realm.json b/stack_orchestrator/data/config/mainnet-eth-keycloak/import/cerc-realm.json index b6b6b606..d7d82361 100644 --- a/stack_orchestrator/data/config/mainnet-eth-keycloak/import/cerc-realm.json +++ b/stack_orchestrator/data/config/mainnet-eth-keycloak/import/cerc-realm.json @@ -2388,4 +2388,4 @@ "clientPolicies": { "policies": [] } -} \ No newline at end of file +} diff --git a/stack_orchestrator/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py b/stack_orchestrator/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py index 86a90180..9c4bd78e 100755 --- a/stack_orchestrator/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py +++ b/stack_orchestrator/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py @@ -12,7 +12,10 @@ from fabric import Connection def dump_src_db_to_file(db_host, db_port, db_user, db_password, db_name, file_name): - command = f"pg_dump -h {db_host} -p {db_port} -U {db_user} -d {db_name} -c --inserts -f {file_name}" + command = ( + f"pg_dump -h {db_host} -p {db_port} -U {db_user} " + f"-d {db_name} -c --inserts -f {file_name}" + ) my_env = os.environ.copy() my_env["PGPASSWORD"] = db_password print(f"Exporting from {db_host}:{db_port}/{db_name} to {file_name}... ", end="") diff --git a/stack_orchestrator/data/config/mainnet-go-opera/go-opera.env b/stack_orchestrator/data/config/mainnet-go-opera/go-opera.env index 8b137891..e69de29b 100644 --- a/stack_orchestrator/data/config/mainnet-go-opera/go-opera.env +++ b/stack_orchestrator/data/config/mainnet-go-opera/go-opera.env @@ -1 +0,0 @@ - diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json index 2bdf04c8..f147ea42 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json @@ -1901,4 +1901,4 @@ "uid": "b54352dd-35f6-4151-97dc-265bab0c67e9", "version": 18, "weekStart": "" -} \ No newline at end of file +} diff --git a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml index c1c3e8e2..b39e6c61 100644 --- a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml +++ b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml @@ -849,7 +849,7 @@ groups: annotations: summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false - + # Secured Finance - uid: secured_finance_diff_external title: secured_finance_watcher_head_tracking diff --git a/stack_orchestrator/data/config/ponder/deploy-erc20-contract.sh b/stack_orchestrator/data/config/ponder/deploy-erc20-contract.sh index ea8111bb..20a5a916 100755 --- a/stack_orchestrator/data/config/ponder/deploy-erc20-contract.sh +++ b/stack_orchestrator/data/config/ponder/deploy-erc20-contract.sh @@ -14,7 +14,7 @@ echo ACCOUNT_PRIVATE_KEY=${CERC_PRIVATE_KEY_DEPLOYER} >> .env if [ -f ${erc20_address_file} ]; then echo "${erc20_address_file} already exists, skipping ERC20 contract deployment" cat ${erc20_address_file} - + # Keep the container running tail -f fi diff --git a/stack_orchestrator/data/config/watcher-mobymask/mobymask-watcher-db.sql b/stack_orchestrator/data/config/watcher-mobymask/mobymask-watcher-db.sql index c77542bd..640dcceb 100644 --- a/stack_orchestrator/data/config/watcher-mobymask/mobymask-watcher-db.sql +++ b/stack_orchestrator/data/config/watcher-mobymask/mobymask-watcher-db.sql @@ -940,4 +940,3 @@ ALTER TABLE ONLY public.state -- -- PostgreSQL database dump complete -- - diff --git a/stack_orchestrator/data/container-build/cerc-builder-gerbil/README.md b/stack_orchestrator/data/container-build/cerc-builder-gerbil/README.md index 854139b9..ae831c94 100644 --- a/stack_orchestrator/data/container-build/cerc-builder-gerbil/README.md +++ b/stack_orchestrator/data/container-build/cerc-builder-gerbil/README.md @@ -18,4 +18,3 @@ root@7c4124bb09e3:/src# ``` Now gerbil commands can be run. - diff --git a/stack_orchestrator/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh b/stack_orchestrator/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh index 95e19194..b84c5d8d 100755 --- a/stack_orchestrator/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh +++ b/stack_orchestrator/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh @@ -23,7 +23,7 @@ local_npm_registry_url=$2 versioned_target_package=$(yarn list --pattern ${target_package} --depth=0 --json --non-interactive --no-progress | jq -r '.data.trees[].name') # Use yarn info to get URL checksums etc from the new registry yarn_info_output=$(yarn info --json $versioned_target_package 2>/dev/null) -# First check if the target version actually exists. +# First check if the target version actually exists. # If it doesn't exist there will be no .data.dist.tarball element, # and jq will output the string "null" package_tarball=$(echo $yarn_info_output | jq -r .data.dist.tarball) diff --git a/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py index 1e6d10f4..4e74e1df 100644 --- a/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py +++ b/stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py @@ -11,6 +11,8 @@ if len(sys.argv) > 1: with open(testnet_config_path) as stream: data = yaml.safe_load(stream) -for key, value in data['el_premine'].items(): - acct = w3.eth.account.from_mnemonic(data['mnemonic'], account_path=key, passphrase='') +for key, value in data["el_premine"].items(): + acct = w3.eth.account.from_mnemonic( + data["mnemonic"], account_path=key, passphrase="" + ) print("%s,%s,%s" % (key, acct.address, acct.key.hex())) diff --git a/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/foundry.toml b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/foundry.toml index 19903e0b..6c7a965b 100644 --- a/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/foundry.toml +++ b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/foundry.toml @@ -4,4 +4,4 @@ out = 'out' libs = ['lib'] remappings = ['ds-test/=lib/ds-test/src/'] -# See more config options https://github.com/gakonst/foundry/tree/master/config \ No newline at end of file +# See more config options https://github.com/gakonst/foundry/tree/master/config diff --git a/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/src/Stateful.sol b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/src/Stateful.sol index 137f9a5a..cfcbb96a 100644 --- a/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/src/Stateful.sol +++ b/stack_orchestrator/data/container-build/cerc-go-ethereum-foundry/stateful/src/Stateful.sol @@ -20,4 +20,4 @@ contract Stateful { function inc() public { x = x + 1; } -} \ No newline at end of file +} diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml index a467903e..1e76d31f 100644 --- a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml @@ -11,4 +11,4 @@ record: foo: bar tags: - a - - b + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml index 3afbd64d..975da6a5 100644 --- a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml @@ -9,4 +9,4 @@ record: foo: bar tags: - a - - b + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconicd/build.sh b/stack_orchestrator/data/container-build/cerc-laconicd/build.sh index 3d897446..85f6aa6b 100755 --- a/stack_orchestrator/data/container-build/cerc-laconicd/build.sh +++ b/stack_orchestrator/data/container-build/cerc-laconicd/build.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Build cerc/laconicd source ${CERC_CONTAINER_BASE_DIR}/build-base.sh -docker build -t cerc/laconicd:local ${build_command_args} ${CERC_REPO_BASE_DIR}/laconicd \ No newline at end of file +docker build -t cerc/laconicd:local ${build_command_args} ${CERC_REPO_BASE_DIR}/laconicd diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index 2c3d2b11..d28c65e8 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -36,7 +36,7 @@ if [ -f "./run-webapp.sh" ]; then ./run-webapp.sh & tpid=$! wait $tpid -else +else "$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r mv .next .next.old mv .next-r/.next . diff --git a/stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile.base b/stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile.base index 62572c47..816997fe 100644 --- a/stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile.base +++ b/stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile.base @@ -5,4 +5,3 @@ WORKDIR /app COPY . . RUN yarn - diff --git a/stack_orchestrator/data/container-build/cerc-ping-pub/scripts/update-explorer-config.sh b/stack_orchestrator/data/container-build/cerc-ping-pub/scripts/update-explorer-config.sh index 812fa5bd..d7f575ce 100755 --- a/stack_orchestrator/data/container-build/cerc-ping-pub/scripts/update-explorer-config.sh +++ b/stack_orchestrator/data/container-build/cerc-ping-pub/scripts/update-explorer-config.sh @@ -22,7 +22,7 @@ fi # infers the directory from which to load chain configuration files # by the presence or absense of the substring "testnet" in the host name # (browser side -- the host name of the host in the address bar of the browser) -# Accordingly we configure our network in both directories in order to +# Accordingly we configure our network in both directories in order to # subvert this lunacy. explorer_mainnet_config_dir=/app/chains/mainnet explorer_testnet_config_dir=/app/chains/testnet diff --git a/stack_orchestrator/data/container-build/cerc-test-database-client/build.sh b/stack_orchestrator/data/container-build/cerc-test-database-client/build.sh index f9a9051f..60a06031 100755 --- a/stack_orchestrator/data/container-build/cerc-test-database-client/build.sh +++ b/stack_orchestrator/data/container-build/cerc-test-database-client/build.sh @@ -2,4 +2,4 @@ # Build cerc/test-container source ${CERC_CONTAINER_BASE_DIR}/build-base.sh SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/test-database-client:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR \ No newline at end of file +docker build -t cerc/test-database-client:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh index 3a114ee0..418c3355 100755 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh @@ -8,7 +8,7 @@ CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}" CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}" CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}" -if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then +if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then # If there is only one HTML file, assume an SPA. if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ]; then CERC_SINGLE_PAGE_APP=true diff --git a/stack_orchestrator/data/k8s/components/ingress/ingress-caddy-kind-deploy.yaml b/stack_orchestrator/data/k8s/components/ingress/ingress-caddy-kind-deploy.yaml new file mode 100644 index 00000000..632dcc05 --- /dev/null +++ b/stack_orchestrator/data/k8s/components/ingress/ingress-caddy-kind-deploy.yaml @@ -0,0 +1,260 @@ +# Caddy Ingress Controller for kind +# Based on: https://github.com/caddyserver/ingress +# Provides automatic HTTPS with Let's Encrypt +apiVersion: v1 +kind: Namespace +metadata: + name: caddy-system + labels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: caddy-ingress-controller + namespace: caddy-system + labels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: caddy-ingress-controller + labels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - namespaces + - services + verbs: + - list + - watch + - get + - apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch + - get + - create + - update + - delete + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: caddy-ingress-controller + labels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: caddy-ingress-controller +subjects: + - kind: ServiceAccount + name: caddy-ingress-controller + namespace: caddy-system +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: caddy-ingress-controller-configmap + namespace: caddy-system + labels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress +data: + # Caddy global options + acmeCA: "https://acme-v02.api.letsencrypt.org/directory" + email: "" +--- +apiVersion: v1 +kind: Service +metadata: + name: caddy-ingress-controller + namespace: caddy-system + labels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress + app.kubernetes.io/component: controller +spec: + type: NodePort + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + - name: https + port: 443 + targetPort: https + protocol: TCP + selector: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress + app.kubernetes.io/component: controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: caddy-ingress-controller + namespace: caddy-system + labels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress + app.kubernetes.io/component: controller +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress + app.kubernetes.io/component: controller + template: + metadata: + labels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress + app.kubernetes.io/component: controller + spec: + serviceAccountName: caddy-ingress-controller + terminationGracePeriodSeconds: 60 + nodeSelector: + ingress-ready: "true" + kubernetes.io/os: linux + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + containers: + - name: caddy-ingress-controller + image: caddy/ingress:latest + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 80 + hostPort: 80 + protocol: TCP + - name: https + containerPort: 443 + hostPort: 443 + protocol: TCP + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - -config-map=caddy-system/caddy-ingress-controller-configmap + - -class-name=caddy + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1000m + memory: 512Mi + readinessProbe: + httpGet: + path: /healthz + port: 9765 + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: 9765 + initialDelaySeconds: 3 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: caddy-data + mountPath: /data + - name: caddy-config + mountPath: /config + volumes: + - name: caddy-data + emptyDir: {} + - name: caddy-config + emptyDir: {} +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: caddy + labels: + app.kubernetes.io/name: caddy-ingress-controller + app.kubernetes.io/instance: caddy-ingress + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +spec: + controller: caddy.io/ingress-controller diff --git a/stack_orchestrator/data/stacks/build-support/README.md b/stack_orchestrator/data/stacks/build-support/README.md index 39bd7dca..ac056d4c 100644 --- a/stack_orchestrator/data/stacks/build-support/README.md +++ b/stack_orchestrator/data/stacks/build-support/README.md @@ -6,7 +6,7 @@ JS/TS/NPM builds need an npm registry to store intermediate package artifacts. This can be supplied by the user (e.g. using a hosted registry or even npmjs.com), or a local registry using gitea can be deployed by stack orchestrator. To use a user-supplied registry set these environment variables: -`CERC_NPM_REGISTRY_URL` and +`CERC_NPM_REGISTRY_URL` and `CERC_NPM_AUTH_TOKEN` Leave `CERC_NPM_REGISTRY_URL` un-set to use the local gitea registry. @@ -22,7 +22,7 @@ $ laconic-so --stack build-support build-containers ``` $ laconic-so --stack package-registry setup-repositories -$ laconic-so --stack package-registry build-containers +$ laconic-so --stack package-registry build-containers $ laconic-so --stack package-registry deploy up [+] Running 3/3 ⠿ Network laconic-aecc4a21d3a502b14522db97d427e850_gitea Created 0.0s diff --git a/stack_orchestrator/data/stacks/fixturenet-blast/stack.yml b/stack_orchestrator/data/stacks/fixturenet-blast/stack.yml index 5b0820c2..a039f3d1 100644 --- a/stack_orchestrator/data/stacks/fixturenet-blast/stack.yml +++ b/stack_orchestrator/data/stacks/fixturenet-blast/stack.yml @@ -14,4 +14,3 @@ containers: pods: - fixturenet-blast - foundry - \ No newline at end of file diff --git a/stack_orchestrator/data/stacks/fixturenet-eth-loaded/README.md b/stack_orchestrator/data/stacks/fixturenet-eth-loaded/README.md index 08ae1e37..d50ca47e 100644 --- a/stack_orchestrator/data/stacks/fixturenet-eth-loaded/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-eth-loaded/README.md @@ -3,4 +3,3 @@ A "loaded" version of fixturenet-eth, with all the bells and whistles enabled. TODO: write me - diff --git a/stack_orchestrator/data/stacks/fixturenet-laconicd/README.md b/stack_orchestrator/data/stacks/fixturenet-laconicd/README.md index d939d0a4..17f96f6f 100644 --- a/stack_orchestrator/data/stacks/fixturenet-laconicd/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-laconicd/README.md @@ -12,7 +12,7 @@ $ chmod +x ./laconic-so $ export PATH=$PATH:$(pwd) # Or move laconic-so to ~/bin or your favorite on-path directory ``` ## 2. Prepare the local build environment -Note that this step needs only to be done once on a new machine. +Note that this step needs only to be done once on a new machine. Detailed instructions can be found [here](../build-support/README.md). For the impatient run these commands: ``` $ laconic-so --stack build-support build-containers --exclude cerc/builder-gerbil diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/README.md b/stack_orchestrator/data/stacks/fixturenet-optimism/README.md index dd681aa5..9c7aa491 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/README.md @@ -52,7 +52,7 @@ laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections. Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. -In addition, a stack-wide port mapping "recipe" can be applied at the time the +In addition, a stack-wide port mapping "recipe" can be applied at the time the `laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: | Recipe | Host Port Mapping | |--------|-------------------| @@ -62,11 +62,11 @@ In addition, a stack-wide port mapping "recipe" can be applied at the time the | localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)| | any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) | -For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `fixturenet-eth-geth-1` RPC to port 8545 and the `op-geth` RPC to port 9545 on the host. +For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `fixturenet-eth-geth-1` RPC to port 8545 and the `op-geth` RPC to port 9545 on the host. Or, you may wish to use `any-same` for the initial mappings -- in which case you'll have to edit the spec to file to ensure the various geth instances aren't all trying to publish to host ports 8545/8546 at once. -### Data volumes +### Data volumes Container data volumes are bind-mounted to specified paths in the host filesystem. The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. @@ -101,7 +101,7 @@ docker logs -f ## Example: bridge some ETH from L1 to L2 -Send some ETH from the desired account to the `L1StandardBridgeProxy` contract on L1 to test bridging to L2. +Send some ETH from the desired account to the `L1StandardBridgeProxy` contract on L1 to test bridging to L2. We can use the testing account `0xe6CE22afe802CAf5fF7d3845cec8c736ecc8d61F` which is pre-funded and unlocked, and the `cerc/foundry:local` container to make use of the `cast` cli. diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py index 2e67da77..bbec0f81 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py @@ -17,16 +17,21 @@ from stack_orchestrator.deploy.deployment_context import DeploymentContext def create(context: DeploymentContext, extra_args): - # Slightly modify the base fixturenet-eth compose file to replace the startup script for fixturenet-eth-geth-1 - # We need to start geth with the flag to allow non eip-155 compliant transactions in order to publish the - # deterministic-deployment-proxy contract, which itself is a prereq for Optimism contract deployment - fixturenet_eth_compose_file = context.deployment_dir.joinpath('compose', 'docker-compose-fixturenet-eth.yml') + # Slightly modify the base fixturenet-eth compose file to replace the + # startup script for fixturenet-eth-geth-1 + # We need to start geth with the flag to allow non eip-155 compliant + # transactions in order to publish the + # deterministic-deployment-proxy contract, which itself is a prereq for + # Optimism contract deployment + fixturenet_eth_compose_file = context.deployment_dir.joinpath( + "compose", "docker-compose-fixturenet-eth.yml" + ) - new_script = '../config/fixturenet-optimism/run-geth.sh:/opt/testnet/run.sh' + new_script = "../config/fixturenet-optimism/run-geth.sh:/opt/testnet/run.sh" def add_geth_volume(yaml_data): - if new_script not in yaml_data['services']['fixturenet-eth-geth-1']['volumes']: - yaml_data['services']['fixturenet-eth-geth-1']['volumes'].append(new_script) + if new_script not in yaml_data["services"]["fixturenet-eth-geth-1"]["volumes"]: + yaml_data["services"]["fixturenet-eth-geth-1"]["volumes"].append(new_script) context.modify_yaml(fixturenet_eth_compose_file, add_geth_volume) diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md b/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md index 4299ca8d..554e28a6 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md @@ -38,7 +38,7 @@ laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections. Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. -In addition, a stack-wide port mapping "recipe" can be applied at the time the +In addition, a stack-wide port mapping "recipe" can be applied at the time the `laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: | Recipe | Host Port Mapping | |--------|-------------------| @@ -48,9 +48,9 @@ In addition, a stack-wide port mapping "recipe" can be applied at the time the | localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)| | any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) | -For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `op-geth` RPC to an easy to remember port like 8545 or 9545 on the host. +For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `op-geth` RPC to an easy to remember port like 8545 or 9545 on the host. -### Data volumes +### Data volumes Container data volumes are bind-mounted to specified paths in the host filesystem. The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. diff --git a/stack_orchestrator/data/stacks/fixturenet-payments/ponder-demo.md b/stack_orchestrator/data/stacks/fixturenet-payments/ponder-demo.md index 2ea416b6..bec58cf9 100644 --- a/stack_orchestrator/data/stacks/fixturenet-payments/ponder-demo.md +++ b/stack_orchestrator/data/stacks/fixturenet-payments/ponder-demo.md @@ -128,7 +128,7 @@ Stack components: removed topics transactionHash - transactionIndex + transactionIndex } getEthBlock( @@ -211,14 +211,14 @@ Stack components: hash } log { - id + id } block { number } } metadata { - pageEndsAtTimestamp + pageEndsAtTimestamp isLastPage } } @@ -227,7 +227,7 @@ Stack components: * Open watcher Ponder app endpoint http://localhost:42069 * Try GQL query to see transfer events - + ```graphql { transferEvents (orderBy: "timestamp", orderDirection: "desc") { @@ -251,9 +251,9 @@ Stack components: ```bash export TOKEN_ADDRESS=$(docker exec payments-ponder-er20-contracts-1 jq -r '.address' ./deployment/erc20-address.json) ``` - + * Transfer token - + ```bash docker exec -it payments-ponder-er20-contracts-1 bash -c "yarn token:transfer:docker --token ${TOKEN_ADDRESS} --to 0xe22AD83A0dE117bA0d03d5E94Eb4E0d80a69C62a --amount 5000" ``` diff --git a/stack_orchestrator/data/stacks/fixturenet-pocket/README.md b/stack_orchestrator/data/stacks/fixturenet-pocket/README.md index a818eaa0..14243e51 100644 --- a/stack_orchestrator/data/stacks/fixturenet-pocket/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-pocket/README.md @@ -48,7 +48,7 @@ or see the full logs: $ laconic-so --stack fixturenet-pocket deploy logs pocket ``` ## 5. Send a relay request to Pocket node -The Pocket node serves relay requests at `http://localhost:8081/v1/client/sim` +The Pocket node serves relay requests at `http://localhost:8081/v1/client/sim` Example request: ``` diff --git a/stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/README.md b/stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/README.md index b1151271..97e1026d 100644 --- a/stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/README.md @@ -154,12 +154,12 @@ http://127.0.0.1:/subgraphs/name/sushiswap/v3-lotus/graphql deployment hasIndexingErrors } - + factories { poolCount id } - + pools { id token0 { diff --git a/stack_orchestrator/data/stacks/graph-node/deploy-subgraph.md b/stack_orchestrator/data/stacks/graph-node/deploy-subgraph.md index 73e200b2..bfc7ad7b 100644 --- a/stack_orchestrator/data/stacks/graph-node/deploy-subgraph.md +++ b/stack_orchestrator/data/stacks/graph-node/deploy-subgraph.md @@ -7,7 +7,7 @@ We will use the [ethereum-gravatar](https://github.com/graphprotocol/graph-tooli - Clone the repo ```bash git clone git@github.com:graphprotocol/graph-tooling.git - + cd graph-tooling ``` @@ -54,11 +54,11 @@ The following steps should be similar for every subgraph - Create and deploy the subgraph ```bash pnpm graph create example --node - + pnpm graph deploy example --ipfs --node ``` - `GRAPH_NODE_DEPLOY_ENDPOINT` and `GRAPH_NODE_IPFS_ENDPOINT` will be available after graph-node has been deployed - - More details can be seen in [Create a deployment](./README.md#create-a-deployment) section + - More details can be seen in [Create a deployment](./README.md#create-a-deployment) section - The subgraph GQL endpoint will be seen after deploy command runs successfully diff --git a/stack_orchestrator/data/stacks/kubo/stack.yml b/stack_orchestrator/data/stacks/kubo/stack.yml index 2552df38..4bfef57a 100644 --- a/stack_orchestrator/data/stacks/kubo/stack.yml +++ b/stack_orchestrator/data/stacks/kubo/stack.yml @@ -1,7 +1,7 @@ version: "1.0" -name: kubo +name: kubo description: "Run kubo (IPFS)" repos: containers: pods: - - kubo + - kubo diff --git a/stack_orchestrator/data/stacks/laconic-dot-com/README.md b/stack_orchestrator/data/stacks/laconic-dot-com/README.md index 3e7ad328..7b8d5632 100644 --- a/stack_orchestrator/data/stacks/laconic-dot-com/README.md +++ b/stack_orchestrator/data/stacks/laconic-dot-com/README.md @@ -2,7 +2,7 @@ ``` laconic-so --stack laconic-dot-com setup-repositories -laconic-so --stack laconic-dot-com build-containers +laconic-so --stack laconic-dot-com build-containers laconic-so --stack laconic-dot-com deploy init --output laconic-website-spec.yml --map-ports-to-host localhost-same laconic-so --stack laconic-dot-com deploy create --spec-file laconic-website-spec.yml --deployment-dir lx-website laconic-so deployment --dir lx-website start diff --git a/stack_orchestrator/data/stacks/lasso/README.md b/stack_orchestrator/data/stacks/lasso/README.md index 226e4e39..ae4328a8 100644 --- a/stack_orchestrator/data/stacks/lasso/README.md +++ b/stack_orchestrator/data/stacks/lasso/README.md @@ -2,6 +2,6 @@ ``` laconic-so --stack lasso setup-repositories -laconic-so --stack lasso build-containers +laconic-so --stack lasso build-containers laconic-so --stack lasso deploy up ``` diff --git a/stack_orchestrator/data/stacks/mainnet-blast/deploy/commands.py b/stack_orchestrator/data/stacks/mainnet-blast/deploy/commands.py index 00aa6970..6d3b32d4 100644 --- a/stack_orchestrator/data/stacks/mainnet-blast/deploy/commands.py +++ b/stack_orchestrator/data/stacks/mainnet-blast/deploy/commands.py @@ -22,18 +22,24 @@ import yaml def create(context, extra_args): # Our goal here is just to copy the json files for blast yml_path = context.deployment_dir.joinpath("spec.yml") - with open(yml_path, 'r') as file: + with open(yml_path, "r") as file: data = yaml.safe_load(file) - mount_point = data['volumes']['blast-data'] + mount_point = data["volumes"]["blast-data"] if mount_point[0] == "/": deploy_dir = Path(mount_point) else: deploy_dir = context.deployment_dir.joinpath(mount_point) command_context = extra_args[2] - compose_file = [f for f in command_context.cluster_context.compose_files if "mainnet-blast" in f][0] - source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "genesis.json") + compose_file = [ + f for f in command_context.cluster_context.compose_files if "mainnet-blast" in f + ][0] + source_config_file = Path(compose_file).parent.parent.joinpath( + "config", "mainnet-blast", "genesis.json" + ) copy(source_config_file, deploy_dir) - source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "rollup.json") + source_config_file = Path(compose_file).parent.parent.joinpath( + "config", "mainnet-blast", "rollup.json" + ) copy(source_config_file, deploy_dir) diff --git a/stack_orchestrator/data/stacks/mainnet-eth-plugeth/README.md b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/README.md index 8ed6bebb..bc92b719 100644 --- a/stack_orchestrator/data/stacks/mainnet-eth-plugeth/README.md +++ b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/README.md @@ -92,7 +92,7 @@ volumes: mainnet_eth_plugeth_geth_1_data: ./data/mainnet_eth_plugeth_geth_1_data mainnet_eth_plugeth_lighthouse_1_data: ./data/mainnet_eth_plugeth_lighthouse_1_data ``` -In addition, a stack-wide port mapping "recipe" can be applied at the time the +In addition, a stack-wide port mapping "recipe" can be applied at the time the `laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: | Recipe | Host Port Mapping | |--------|-------------------| diff --git a/stack_orchestrator/data/stacks/mainnet-eth-plugeth/deploy/commands.py b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/deploy/commands.py index 5aba9547..b7a7e002 100644 --- a/stack_orchestrator/data/stacks/mainnet-eth-plugeth/deploy/commands.py +++ b/stack_orchestrator/data/stacks/mainnet-eth-plugeth/deploy/commands.py @@ -27,6 +27,8 @@ def setup(ctx): def create(ctx, extra_args): # Generate the JWT secret and save to its config file secret = token_hex(32) - jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_plugeth_config_data", "jwtsecret") - with open(jwt_file_path, 'w+') as jwt_file: + jwt_file_path = ctx.deployment_dir.joinpath( + "data", "mainnet_eth_plugeth_config_data", "jwtsecret" + ) + with open(jwt_file_path, "w+") as jwt_file: jwt_file.write(secret) diff --git a/stack_orchestrator/data/stacks/mainnet-eth/README.md b/stack_orchestrator/data/stacks/mainnet-eth/README.md index 2656b75b..586b1067 100644 --- a/stack_orchestrator/data/stacks/mainnet-eth/README.md +++ b/stack_orchestrator/data/stacks/mainnet-eth/README.md @@ -92,7 +92,7 @@ volumes: mainnet_eth_geth_1_data: ./data/mainnet_eth_geth_1_data mainnet_eth_lighthouse_1_data: ./data/mainnet_eth_lighthouse_1_data ``` -In addition, a stack-wide port mapping "recipe" can be applied at the time the +In addition, a stack-wide port mapping "recipe" can be applied at the time the `laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: | Recipe | Host Port Mapping | |--------|-------------------| diff --git a/stack_orchestrator/data/stacks/mainnet-eth/deploy/commands.py b/stack_orchestrator/data/stacks/mainnet-eth/deploy/commands.py index 9fcecbcf..545e16a1 100644 --- a/stack_orchestrator/data/stacks/mainnet-eth/deploy/commands.py +++ b/stack_orchestrator/data/stacks/mainnet-eth/deploy/commands.py @@ -27,6 +27,8 @@ def setup(ctx): def create(ctx, extra_args): # Generate the JWT secret and save to its config file secret = token_hex(32) - jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_config_data", "jwtsecret") - with open(jwt_file_path, 'w+') as jwt_file: + jwt_file_path = ctx.deployment_dir.joinpath( + "data", "mainnet_eth_config_data", "jwtsecret" + ) + with open(jwt_file_path, "w+") as jwt_file: jwt_file.write(secret) diff --git a/stack_orchestrator/data/stacks/mainnet-go-opera/README.md b/stack_orchestrator/data/stacks/mainnet-go-opera/README.md index f5e761eb..955a692b 100644 --- a/stack_orchestrator/data/stacks/mainnet-go-opera/README.md +++ b/stack_orchestrator/data/stacks/mainnet-go-opera/README.md @@ -36,9 +36,9 @@ laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | 'mainnet-109331-no-histor laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.034] Maximum peer count total=50 laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.034] Smartcard socket not found, disabling err="stat /run/pcscd/pcscd.comm: no such file or directory" laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.034] Genesis file is a known preset name="Mainnet-109331 without history" -laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.052] Applying genesis state -laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.052] - Reading epochs unit 0 -laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.054] - Reading blocks unit 0 +laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.052] Applying genesis state +laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.052] - Reading epochs unit 0 +laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.054] - Reading blocks unit 0 laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.530] Applied genesis state name=main id=250 genesis=0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4 laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.531] Regenerated local transaction journal transactions=0 accounts=0 laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.532] Starting peer-to-peer node instance=go-opera/v1.1.2-rc.5-50cd051d-1677276206/linux-amd64/go1.19.10 @@ -47,7 +47,7 @@ laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.537] laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.537] IPC endpoint opened url=/root/.opera/opera.ipc laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] HTTP server started endpoint=[::]:18545 prefix= cors=* vhosts=localhost laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] WebSocket enabled url=ws://[::]:18546 -laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] Rebuilding state snapshot +laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] Rebuilding state snapshot laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] EVM snapshot module=gossip-store at=000000..000000 generating=true laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] Resuming state snapshot generation accounts=0 slots=0 storage=0.00B elapsed="189.74µs" laconic-f028f14527b95e2eb97f0c0229d00939-go-opera-1 | INFO [06-20|13:32:33.538] Generated state snapshot accounts=0 slots=0 storage=0.00B elapsed="265.061µs" diff --git a/stack_orchestrator/data/stacks/mainnet-laconic/README.md b/stack_orchestrator/data/stacks/mainnet-laconic/README.md index 67984e5b..8407d70c 100644 --- a/stack_orchestrator/data/stacks/mainnet-laconic/README.md +++ b/stack_orchestrator/data/stacks/mainnet-laconic/README.md @@ -1,2 +1 @@ # Laconic Mainnet Deployment (experimental) - diff --git a/stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py b/stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py index b3ce32d3..9364a9c8 100644 --- a/stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py +++ b/stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py @@ -14,7 +14,10 @@ # along with this program. If not, see . from stack_orchestrator.util import get_yaml -from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand +from stack_orchestrator.deploy.deploy_types import ( + DeployCommandContext, + LaconicStackSetupCommand, +) from stack_orchestrator.deploy.deployment_context import DeploymentContext from stack_orchestrator.deploy.stack_state import State from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command @@ -75,7 +78,12 @@ def _copy_gentx_files(network_dir: Path, gentx_file_list: str): gentx_files = _comma_delimited_to_list(gentx_file_list) for gentx_file in gentx_files: gentx_file_path = Path(gentx_file) - copyfile(gentx_file_path, os.path.join(network_dir, "config", "gentx", os.path.basename(gentx_file_path))) + copyfile( + gentx_file_path, + os.path.join( + network_dir, "config", "gentx", os.path.basename(gentx_file_path) + ), + ) def _remove_persistent_peers(network_dir: Path): @@ -86,8 +94,13 @@ def _remove_persistent_peers(network_dir: Path): with open(config_file_path, "r") as input_file: config_file_content = input_file.read() persistent_peers_pattern = '^persistent_peers = "(.+?)"' - replace_with = "persistent_peers = \"\"" - config_file_content = re.sub(persistent_peers_pattern, replace_with, config_file_content, flags=re.MULTILINE) + replace_with = 'persistent_peers = ""' + config_file_content = re.sub( + persistent_peers_pattern, + replace_with, + config_file_content, + flags=re.MULTILINE, + ) with open(config_file_path, "w") as output_file: output_file.write(config_file_content) @@ -100,8 +113,13 @@ def _insert_persistent_peers(config_dir: Path, new_persistent_peers: str): with open(config_file_path, "r") as input_file: config_file_content = input_file.read() persistent_peers_pattern = r'^persistent_peers = ""' - replace_with = f"persistent_peers = \"{new_persistent_peers}\"" - config_file_content = re.sub(persistent_peers_pattern, replace_with, config_file_content, flags=re.MULTILINE) + replace_with = f'persistent_peers = "{new_persistent_peers}"' + config_file_content = re.sub( + persistent_peers_pattern, + replace_with, + config_file_content, + flags=re.MULTILINE, + ) with open(config_file_path, "w") as output_file: output_file.write(config_file_content) @@ -113,9 +131,11 @@ def _enable_cors(config_dir: Path): sys.exit(1) with open(config_file_path, "r") as input_file: config_file_content = input_file.read() - cors_pattern = r'^cors_allowed_origins = \[]' + cors_pattern = r"^cors_allowed_origins = \[]" replace_with = 'cors_allowed_origins = ["*"]' - config_file_content = re.sub(cors_pattern, replace_with, config_file_content, flags=re.MULTILINE) + config_file_content = re.sub( + cors_pattern, replace_with, config_file_content, flags=re.MULTILINE + ) with open(config_file_path, "w") as output_file: output_file.write(config_file_content) app_file_path = config_dir.joinpath("app.toml") @@ -124,9 +144,11 @@ def _enable_cors(config_dir: Path): sys.exit(1) with open(app_file_path, "r") as input_file: app_file_content = input_file.read() - cors_pattern = r'^enabled-unsafe-cors = false' + cors_pattern = r"^enabled-unsafe-cors = false" replace_with = "enabled-unsafe-cors = true" - app_file_content = re.sub(cors_pattern, replace_with, app_file_content, flags=re.MULTILINE) + app_file_content = re.sub( + cors_pattern, replace_with, app_file_content, flags=re.MULTILINE + ) with open(app_file_path, "w") as output_file: output_file.write(app_file_content) @@ -141,7 +163,9 @@ def _set_listen_address(config_dir: Path): existing_pattern = r'^laddr = "tcp://127.0.0.1:26657"' replace_with = 'laddr = "tcp://0.0.0.0:26657"' print(f"Replacing in: {config_file_path}") - config_file_content = re.sub(existing_pattern, replace_with, config_file_content, flags=re.MULTILINE) + config_file_content = re.sub( + existing_pattern, replace_with, config_file_content, flags=re.MULTILINE + ) with open(config_file_path, "w") as output_file: output_file.write(config_file_content) app_file_path = config_dir.joinpath("app.toml") @@ -152,10 +176,14 @@ def _set_listen_address(config_dir: Path): app_file_content = input_file.read() existing_pattern1 = r'^address = "tcp://localhost:1317"' replace_with1 = 'address = "tcp://0.0.0.0:1317"' - app_file_content = re.sub(existing_pattern1, replace_with1, app_file_content, flags=re.MULTILINE) + app_file_content = re.sub( + existing_pattern1, replace_with1, app_file_content, flags=re.MULTILINE + ) existing_pattern2 = r'^address = "localhost:9090"' replace_with2 = 'address = "0.0.0.0:9090"' - app_file_content = re.sub(existing_pattern2, replace_with2, app_file_content, flags=re.MULTILINE) + app_file_content = re.sub( + existing_pattern2, replace_with2, app_file_content, flags=re.MULTILINE + ) with open(app_file_path, "w") as output_file: output_file.write(app_file_content) @@ -164,7 +192,10 @@ def _phase_from_params(parameters): phase = SetupPhase.ILLEGAL if parameters.initialize_network: if parameters.join_network or parameters.create_network: - print("Can't supply --join-network or --create-network with --initialize-network") + print( + "Can't supply --join-network or --create-network " + "with --initialize-network" + ) sys.exit(1) if not parameters.chain_id: print("--chain-id is required") @@ -176,24 +207,36 @@ def _phase_from_params(parameters): phase = SetupPhase.INITIALIZE elif parameters.join_network: if parameters.initialize_network or parameters.create_network: - print("Can't supply --initialize-network or --create-network with --join-network") + print( + "Can't supply --initialize-network or --create-network " + "with --join-network" + ) sys.exit(1) phase = SetupPhase.JOIN elif parameters.create_network: if parameters.initialize_network or parameters.join_network: - print("Can't supply --initialize-network or --join-network with --create-network") + print( + "Can't supply --initialize-network or --join-network " + "with --create-network" + ) sys.exit(1) phase = SetupPhase.CREATE elif parameters.connect_network: if parameters.initialize_network or parameters.join_network: - print("Can't supply --initialize-network or --join-network with --connect-network") + print( + "Can't supply --initialize-network or --join-network " + "with --connect-network" + ) sys.exit(1) phase = SetupPhase.CONNECT return phase -def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args): - +def setup( + command_context: DeployCommandContext, + parameters: LaconicStackSetupCommand, + extra_args, +): options = opts.o currency = "alnt" # Does this need to be a parameter? @@ -205,12 +248,9 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo network_dir = Path(parameters.network_dir).absolute() laconicd_home_path_in_container = "/laconicd-home" - mounts = [ - VolumeMapping(network_dir, laconicd_home_path_in_container) - ] + mounts = [VolumeMapping(str(network_dir), laconicd_home_path_in_container)] if phase == SetupPhase.INITIALIZE: - # We want to create the directory so if it exists that's an error if os.path.exists(network_dir): print(f"Error: network directory {network_dir} already exists") @@ -220,13 +260,18 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo output, status = run_container_command( command_context, - "laconicd", f"laconicd init {parameters.node_moniker} --home {laconicd_home_path_in_container}\ - --chain-id {parameters.chain_id} --default-denom {currency}", mounts) + "laconicd", + f"laconicd init {parameters.node_moniker} " + f"--home {laconicd_home_path_in_container} " + f"--chain-id {parameters.chain_id} --default-denom {currency}", + mounts, + ) if options.debug: print(f"Command output: {output}") elif phase == SetupPhase.JOIN: - # In the join phase (alternative to connect) we are participating in a genesis ceremony for the chain + # In the join phase (alternative to connect) we are participating in a + # genesis ceremony for the chain if not os.path.exists(network_dir): print(f"Error: network directory {network_dir} doesn't exist") sys.exit(1) @@ -234,52 +279,72 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo chain_id = _get_chain_id_from_config(network_dir) output1, status1 = run_container_command( - command_context, "laconicd", f"laconicd keys add {parameters.key_name} --home {laconicd_home_path_in_container}\ - --keyring-backend test", mounts) + command_context, + "laconicd", + f"laconicd keys add {parameters.key_name} " + f"--home {laconicd_home_path_in_container} --keyring-backend test", + mounts, + ) if options.debug: print(f"Command output: {output1}") output2, status2 = run_container_command( command_context, "laconicd", - f"laconicd genesis add-genesis-account {parameters.key_name} 12900000000000000000000{currency}\ - --home {laconicd_home_path_in_container} --keyring-backend test", - mounts) + f"laconicd genesis add-genesis-account {parameters.key_name} " + f"12900000000000000000000{currency} " + f"--home {laconicd_home_path_in_container} --keyring-backend test", + mounts, + ) if options.debug: print(f"Command output: {output2}") output3, status3 = run_container_command( command_context, "laconicd", - f"laconicd genesis gentx {parameters.key_name} 90000000000{currency} --home {laconicd_home_path_in_container}\ - --chain-id {chain_id} --keyring-backend test", - mounts) + f"laconicd genesis gentx {parameters.key_name} " + f"90000000000{currency} --home {laconicd_home_path_in_container} " + f"--chain-id {chain_id} --keyring-backend test", + mounts, + ) if options.debug: print(f"Command output: {output3}") output4, status4 = run_container_command( command_context, "laconicd", - f"laconicd keys show {parameters.key_name} -a --home {laconicd_home_path_in_container} --keyring-backend test", - mounts) + f"laconicd keys show {parameters.key_name} -a " + f"--home {laconicd_home_path_in_container} --keyring-backend test", + mounts, + ) print(f"Node account address: {output4}") elif phase == SetupPhase.CONNECT: - # In the connect phase (named to not conflict with join) we are making a node that syncs a chain with existing genesis.json - # but not with validator role. We need this kind of node in order to bootstrap it into a validator after it syncs + # In the connect phase (named to not conflict with join) we are + # making a node that syncs a chain with existing genesis.json + # but not with validator role. We need this kind of node in order to + # bootstrap it into a validator after it syncs output1, status1 = run_container_command( - command_context, "laconicd", f"laconicd keys add {parameters.key_name} --home {laconicd_home_path_in_container}\ - --keyring-backend test", mounts) + command_context, + "laconicd", + f"laconicd keys add {parameters.key_name} " + f"--home {laconicd_home_path_in_container} --keyring-backend test", + mounts, + ) if options.debug: print(f"Command output: {output1}") output2, status2 = run_container_command( command_context, "laconicd", - f"laconicd keys show {parameters.key_name} -a --home {laconicd_home_path_in_container} --keyring-backend test", - mounts) + f"laconicd keys show {parameters.key_name} -a " + f"--home {laconicd_home_path_in_container} --keyring-backend test", + mounts, + ) print(f"Node account address: {output2}") output3, status3 = run_container_command( command_context, "laconicd", - f"laconicd cometbft show-validator --home {laconicd_home_path_in_container}", - mounts) + f"laconicd cometbft show-validator " + f"--home {laconicd_home_path_in_container}", + mounts, + ) print(f"Node validator address: {output3}") elif phase == SetupPhase.CREATE: @@ -287,42 +352,74 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo print(f"Error: network directory {network_dir} doesn't exist") sys.exit(1) - # In the CREATE phase, we are either a "coordinator" node, generating the genesis.json file ourselves - # OR we are a "not-coordinator" node, consuming a genesis file we got from the coordinator node. + # In the CREATE phase, we are either a "coordinator" node, + # generating the genesis.json file ourselves + # OR we are a "not-coordinator" node, consuming a genesis file from + # the coordinator node. if parameters.genesis_file: # We got the genesis file from elsewhere # Copy it into our network dir genesis_file_path = Path(parameters.genesis_file) if not os.path.exists(genesis_file_path): - print(f"Error: supplied genesis file: {parameters.genesis_file} does not exist.") + print( + f"Error: supplied genesis file: {parameters.genesis_file} " + "does not exist." + ) sys.exit(1) - copyfile(genesis_file_path, os.path.join(network_dir, "config", os.path.basename(genesis_file_path))) + copyfile( + genesis_file_path, + os.path.join( + network_dir, "config", os.path.basename(genesis_file_path) + ), + ) else: # We're generating the genesis file # First look in the supplied gentx files for the other nodes' keys - other_node_keys = _get_node_keys_from_gentx_files(parameters.gentx_address_list) + other_node_keys = _get_node_keys_from_gentx_files( + parameters.gentx_address_list + ) # Add those keys to our genesis, with balances we determine here (why?) + outputk = None for other_node_key in other_node_keys: outputk, statusk = run_container_command( - command_context, "laconicd", f"laconicd genesis add-genesis-account {other_node_key} \ - 12900000000000000000000{currency}\ - --home {laconicd_home_path_in_container} --keyring-backend test", mounts) - if options.debug: + command_context, + "laconicd", + f"laconicd genesis add-genesis-account {other_node_key} " + f"12900000000000000000000{currency} " + f"--home {laconicd_home_path_in_container} " + "--keyring-backend test", + mounts, + ) + if options.debug and outputk is not None: print(f"Command output: {outputk}") # Copy the gentx json files into our network dir _copy_gentx_files(network_dir, parameters.gentx_file_list) # Now we can run collect-gentxs output1, status1 = run_container_command( - command_context, "laconicd", f"laconicd genesis collect-gentxs --home {laconicd_home_path_in_container}", mounts) + command_context, + "laconicd", + f"laconicd genesis collect-gentxs " + f"--home {laconicd_home_path_in_container}", + mounts, + ) if options.debug: print(f"Command output: {output1}") - print(f"Generated genesis file, please copy to other nodes as required: \ - {os.path.join(network_dir, 'config', 'genesis.json')}") - # Last thing, collect-gentxs puts a likely bogus set of persistent_peers in config.toml so we remove that now + genesis_path = os.path.join(network_dir, "config", "genesis.json") + print( + f"Generated genesis file, please copy to other nodes " + f"as required: {genesis_path}" + ) + # Last thing, collect-gentxs puts a likely bogus set of persistent_peers + # in config.toml so we remove that now _remove_persistent_peers(network_dir) # In both cases we validate the genesis file now output2, status1 = run_container_command( - command_context, "laconicd", f"laconicd genesis validate-genesis --home {laconicd_home_path_in_container}", mounts) + command_context, + "laconicd", + f"laconicd genesis validate-genesis " + f"--home {laconicd_home_path_in_container}", + mounts, + ) print(f"validate-genesis result: {output2}") else: @@ -341,15 +438,23 @@ def create(deployment_context: DeploymentContext, extra_args): sys.exit(1) config_dir_path = network_dir_path.joinpath("config") if not (config_dir_path.exists() and config_dir_path.is_dir()): - print(f"Error: supplied network directory does not contain a config directory: {config_dir_path}") + print( + f"Error: supplied network directory does not contain " + f"a config directory: {config_dir_path}" + ) sys.exit(1) data_dir_path = network_dir_path.joinpath("data") if not (data_dir_path.exists() and data_dir_path.is_dir()): - print(f"Error: supplied network directory does not contain a data directory: {data_dir_path}") + print( + f"Error: supplied network directory does not contain " + f"a data directory: {data_dir_path}" + ) sys.exit(1) # Copy the network directory contents into our deployment # TODO: change this to work with non local paths - deployment_config_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-config") + deployment_config_dir = deployment_context.deployment_dir.joinpath( + "data", "laconicd-config" + ) copytree(config_dir_path, deployment_config_dir, dirs_exist_ok=True) # If supplied, add the initial persistent peers to the config file if extra_args[1]: @@ -360,7 +465,9 @@ def create(deployment_context: DeploymentContext, extra_args): _set_listen_address(deployment_config_dir) # Copy the data directory contents into our deployment # TODO: change this to work with non local paths - deployment_data_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-data") + deployment_data_dir = deployment_context.deployment_dir.joinpath( + "data", "laconicd-data" + ) copytree(data_dir_path, deployment_data_dir, dirs_exist_ok=True) diff --git a/stack_orchestrator/data/stacks/mobymask/stack.yml b/stack_orchestrator/data/stacks/mobymask/stack.yml index 22a30a0f..3ce9c625 100644 --- a/stack_orchestrator/data/stacks/mobymask/stack.yml +++ b/stack_orchestrator/data/stacks/mobymask/stack.yml @@ -5,4 +5,4 @@ repos: containers: - cerc/watcher-mobymask pods: - - watcher-mobymask \ No newline at end of file + - watcher-mobymask diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index f0d39be6..4f41daa2 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -180,7 +180,7 @@ Set the following env variables in the deployment env config file (`monitoring-d # (Optional, default: http://localhost:3000) GF_SERVER_ROOT_URL= - + # RPC endpoint used by graph-node for upstream head metric # (Optional, default: https://mainnet.infura.io/v3) GRAPH_NODE_RPC_ENDPOINT= diff --git a/stack_orchestrator/data/stacks/test-database/README.md b/stack_orchestrator/data/stacks/test-database/README.md index 1dcdcc7b..aae68cd4 100644 --- a/stack_orchestrator/data/stacks/test-database/README.md +++ b/stack_orchestrator/data/stacks/test-database/README.md @@ -1,3 +1,3 @@ # Test Database Stack -A stack with a database for test/demo purposes. \ No newline at end of file +A stack with a database for test/demo purposes. diff --git a/stack_orchestrator/data/stacks/test/README.md b/stack_orchestrator/data/stacks/test/README.md index aef333fc..b95adfe7 100644 --- a/stack_orchestrator/data/stacks/test/README.md +++ b/stack_orchestrator/data/stacks/test/README.md @@ -1,3 +1,3 @@ # Test Stack -A stack for test/demo purposes. \ No newline at end of file +A stack for test/demo purposes. diff --git a/stack_orchestrator/data/stacks/test/deploy/commands.py b/stack_orchestrator/data/stacks/test/deploy/commands.py index e6601eae..356338af 100644 --- a/stack_orchestrator/data/stacks/test/deploy/commands.py +++ b/stack_orchestrator/data/stacks/test/deploy/commands.py @@ -15,6 +15,7 @@ from stack_orchestrator.util import get_yaml from stack_orchestrator.deploy.deploy_types import DeployCommandContext +from stack_orchestrator.deploy.deployment_context import DeploymentContext from stack_orchestrator.deploy.stack_state import State from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command from pathlib import Path @@ -24,16 +25,20 @@ default_spec_file_content = """config: """ -# Output a known string to a know file in the bind mounted directory ./container-output-dir +# Output a known string to a know file in the bind mounted directory +# ./container-output-dir # for test purposes -- test checks that the file was written. def setup(command_context: DeployCommandContext, parameters, extra_args): host_directory = "./container-output-dir" host_directory_absolute = Path(extra_args[0]).absolute().joinpath(host_directory) host_directory_absolute.mkdir(parents=True, exist_ok=True) - mounts = [ - VolumeMapping(host_directory_absolute, "/data") - ] - output, status = run_container_command(command_context, "test", "echo output-data > /data/output-file && echo success", mounts) + mounts = [VolumeMapping(str(host_directory_absolute), "/data")] + output, status = run_container_command( + command_context, + "test", + "echo output-data > /data/output-file && echo success", + mounts, + ) def init(command_context: DeployCommandContext): @@ -41,10 +46,10 @@ def init(command_context: DeployCommandContext): return yaml.load(default_spec_file_content) -def create(command_context: DeployCommandContext, extra_args): +def create(deployment_context: DeploymentContext, extra_args): data = "create-command-output-data" - output_file_path = command_context.deployment_dir.joinpath("create-file") - with open(output_file_path, 'w+') as output_file: + output_file_path = deployment_context.deployment_dir.joinpath("create-file") + with open(output_file_path, "w+") as output_file: output_file.write(data) diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index 565fcfa2..c6397aad 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -14,8 +14,13 @@ # along with this program. If not, see . from pathlib import Path +from typing import Optional from python_on_whales import DockerClient, DockerException -from stack_orchestrator.deploy.deployer import Deployer, DeployerException, DeployerConfigGenerator +from stack_orchestrator.deploy.deployer import ( + Deployer, + DeployerException, + DeployerConfigGenerator, +) from stack_orchestrator.deploy.deployment_context import DeploymentContext from stack_orchestrator.opts import opts @@ -24,10 +29,24 @@ class DockerDeployer(Deployer): name: str = "compose" type: str - def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None: - self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name, - compose_env_file=compose_env_file) + def __init__( + self, + type: str, + deployment_context: Optional[DeploymentContext], + compose_files: list, + compose_project_name: Optional[str], + compose_env_file: Optional[str], + ) -> None: + self.docker = DockerClient( + compose_files=compose_files, + compose_project_name=compose_project_name, + compose_env_file=compose_env_file, + ) self.type = type + # Store these for later use in run_job + self.compose_files = compose_files + self.compose_project_name = compose_project_name + self.compose_env_file = compose_env_file def up(self, detach, skip_cluster_management, services): if not opts.o.dry_run: @@ -68,35 +87,98 @@ class DockerDeployer(Deployer): def port(self, service, private_port): if not opts.o.dry_run: try: - return self.docker.compose.port(service=service, private_port=private_port) + return self.docker.compose.port( + service=service, private_port=private_port + ) except DockerException as e: raise DeployerException(e) def execute(self, service, command, tty, envs): if not opts.o.dry_run: try: - return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs) + return self.docker.compose.execute( + service=service, command=command, tty=tty, envs=envs + ) except DockerException as e: raise DeployerException(e) def logs(self, services, tail, follow, stream): if not opts.o.dry_run: try: - return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream) + return self.docker.compose.logs( + services=services, tail=tail, follow=follow, stream=stream + ) except DockerException as e: raise DeployerException(e) - def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): + def run( + self, + image: str, + command=None, + user=None, + volumes=None, + entrypoint=None, + env={}, + ports=[], + detach=False, + ): if not opts.o.dry_run: try: - return self.docker.run(image=image, command=command, user=user, volumes=volumes, - entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0) + return self.docker.run( + image=image, + command=command if command else [], + user=user, + volumes=volumes, + entrypoint=entrypoint, + envs=env, + detach=detach, + publish=ports, + publish_all=len(ports) == 0, + ) + except DockerException as e: + raise DeployerException(e) + + def run_job(self, job_name: str, release_name: Optional[str] = None): + # release_name is ignored for Docker deployments (only used for K8s/Helm) + if not opts.o.dry_run: + try: + # Find job compose file in compose-jobs directory + # The deployment should have compose-jobs/docker-compose-.yml + if not self.compose_files: + raise DeployerException("No compose files configured") + + # Deployment directory is parent of compose directory + compose_dir = Path(self.compose_files[0]).parent + deployment_dir = compose_dir.parent + job_compose_file = ( + deployment_dir / "compose-jobs" / f"docker-compose-{job_name}.yml" + ) + + if not job_compose_file.exists(): + raise DeployerException( + f"Job compose file not found: {job_compose_file}" + ) + + if opts.o.verbose: + print(f"Running job from: {job_compose_file}") + + # Create a DockerClient for the job compose file with same + # project name and env file + # This allows the job to access volumes from the main deployment + job_docker = DockerClient( + compose_files=[job_compose_file], + compose_project_name=self.compose_project_name, + compose_env_file=self.compose_env_file, + ) + + # Run the job with --rm flag to remove container after completion + return job_docker.compose.run(service=job_name, remove=True, tty=True) + except DockerException as e: raise DeployerException(e) class DockerDeployerConfigGenerator(DeployerConfigGenerator): - def __init__(self, type: str) -> None: super().__init__() diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index f8802758..86c1856c 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -21,6 +21,7 @@ import os import sys from dataclasses import dataclass from importlib import resources +from typing import Optional import subprocess import click from pathlib import Path @@ -35,27 +36,36 @@ from stack_orchestrator.util import ( stack_is_in_deployment, resolve_compose_file, ) -from stack_orchestrator.deploy.deployer import Deployer, DeployerException +from stack_orchestrator.deploy.deployer import DeployerException from stack_orchestrator.deploy.deployer_factory import getDeployer +from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext from stack_orchestrator.deploy.deployment_context import DeploymentContext from stack_orchestrator.deploy.deployment_create import create as deployment_create from stack_orchestrator.deploy.deployment_create import init as deployment_init from stack_orchestrator.deploy.deployment_create import setup as deployment_setup +from stack_orchestrator.deploy.k8s import k8s_command @click.group() @click.option("--include", help="only start these components") -@click.option("--exclude", help="don\'t start these components") +@click.option("--exclude", help="don't start these components") @click.option("--env-file", help="env file to be used") @click.option("--cluster", help="specify a non-default cluster name") -@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s or k8s-kind)") +@click.option( + "--deploy-to", help="cluster system to deploy to (compose or k8s or k8s-kind)" +) @click.pass_context def command(ctx, include, exclude, env_file, cluster, deploy_to): - '''deploy a stack''' + """deploy a stack""" - # Although in theory for some subcommands (e.g. deploy create) the stack can be inferred, - # Click doesn't allow us to know that here, so we make providing the stack mandatory + # k8s subcommand doesn't require a stack + if ctx.invoked_subcommand == "k8s": + return + + # Although in theory for some subcommands (e.g. deploy create) the stack + # can be inferred, Click doesn't allow us to know that here, so we make + # providing the stack mandatory stack = global_options2(ctx).stack if not stack: print("Error: --stack option is required") @@ -68,30 +78,65 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to): deploy_to = "compose" stack = get_stack_path(stack) - ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to) - # Subcommand is executed now, by the magic of click - - -def create_deploy_context( - global_context, - deployment_context: DeploymentContext, + ctx.obj = create_deploy_context( + global_options2(ctx), + None, stack, include, exclude, cluster, env_file, - deploy_to) -> DeployCommandContext: + deploy_to, + ) + # Subcommand is executed now, by the magic of click + + +def create_deploy_context( + global_context, + deployment_context: Optional[DeploymentContext], + stack, + include, + exclude, + cluster, + env_file, + deploy_to, +) -> DeployCommandContext: # Extract the cluster name from the deployment, if we have one if deployment_context and cluster is None: cluster = deployment_context.get_cluster_id() - cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file) - deployer = getDeployer(deploy_to, deployment_context, compose_files=cluster_context.compose_files, - compose_project_name=cluster_context.cluster, - compose_env_file=cluster_context.env_file) + + # Check if this is a helm chart deployment (has chart/ but no compose/) + # TODO: Add a new deployment type for helm chart deployments + # To avoid relying on chart existence in such cases + is_helm_chart_deployment = False + if deployment_context: + chart_dir = deployment_context.deployment_dir / "chart" + compose_dir = deployment_context.deployment_dir / "compose" + is_helm_chart_deployment = chart_dir.exists() and not compose_dir.exists() + + # For helm chart deployments, skip compose file loading + if is_helm_chart_deployment: + cluster_context = ClusterContext( + global_context, cluster, [], [], [], None, env_file + ) + else: + cluster_context = _make_cluster_context( + global_context, stack, include, exclude, cluster, env_file + ) + + deployer = getDeployer( + deploy_to, + deployment_context, + compose_files=cluster_context.compose_files, + compose_project_name=cluster_context.cluster, + compose_env_file=cluster_context.env_file, + ) return DeployCommandContext(stack, cluster_context, deployer) -def up_operation(ctx, services_list, stay_attached=False, skip_cluster_management=False): +def up_operation( + ctx, services_list, stay_attached=False, skip_cluster_management=False +): global_context = ctx.parent.parent.obj deploy_context = ctx.obj cluster_context = deploy_context.cluster_context @@ -99,21 +144,38 @@ def up_operation(ctx, services_list, stay_attached=False, skip_cluster_managemen for attr, value in container_exec_env.items(): os.environ[attr] = value if global_context.verbose: - print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}") + print( + f"Running compose up with container_exec_env: {container_exec_env}, " + f"extra_args: {services_list}" + ) for pre_start_command in cluster_context.pre_start_commands: _run_command(global_context, cluster_context.cluster, pre_start_command) - deploy_context.deployer.up(detach=not stay_attached, skip_cluster_management=skip_cluster_management, services=services_list) + deploy_context.deployer.up( + detach=not stay_attached, + skip_cluster_management=skip_cluster_management, + services=services_list, + ) for post_start_command in cluster_context.post_start_commands: _run_command(global_context, cluster_context.cluster, post_start_command) - _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env) + _orchestrate_cluster_config( + global_context, + cluster_context.config, + deploy_context.deployer, + container_exec_env, + ) def down_operation(ctx, delete_volumes, extra_args_list, skip_cluster_management=False): timeout_arg = None if extra_args_list: timeout_arg = extra_args_list[0] - # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully - ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes, skip_cluster_management=skip_cluster_management) + # Specify shutdown timeout (default 10s) to give services enough time to + # shutdown gracefully + ctx.obj.deployer.down( + timeout=timeout_arg, + volumes=delete_volumes, + skip_cluster_management=skip_cluster_management, + ) def status_operation(ctx): @@ -140,7 +202,11 @@ def ps_operation(ctx): if mapping is None: print(f"{port_mapping}", end="") else: - print(f"{mapping[0]['HostIp']}:{mapping[0]['HostPort']}->{port_mapping}", end="") + print( + f"{mapping[0]['HostIp']}:{mapping[0]['HostPort']}" + f"->{port_mapping}", + end="", + ) comma = ", " print() else: @@ -175,7 +241,9 @@ def exec_operation(ctx, extra_args): if global_context.verbose: print(f"Running compose exec {service_name} {command_to_exec}") try: - ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env, tty=True) + ctx.obj.deployer.execute( + service_name, command_to_exec, envs=container_exec_env, tty=True + ) except DeployerException: print("container command returned error exit status") @@ -183,13 +251,26 @@ def exec_operation(ctx, extra_args): def logs_operation(ctx, tail: int, follow: bool, extra_args: str): extra_args_list = list(extra_args) or None services_list = extra_args_list if extra_args_list is not None else [] - logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True) + logs_stream = ctx.obj.deployer.logs( + services=services_list, tail=tail, follow=follow, stream=True + ) for stream_type, stream_content in logs_stream: print(stream_content.decode("utf-8"), end="") +def run_job_operation(ctx, job_name: str, helm_release: Optional[str] = None): + global_context = ctx.parent.parent.obj + if not global_context.dry_run: + print(f"Running job: {job_name}") + try: + ctx.obj.deployer.run_job(job_name, helm_release) + except Exception as e: + print(f"Error running job {job_name}: {e}") + sys.exit(1) + + @command.command() -@click.argument('extra_args', nargs=-1) # help: command: up +@click.argument("extra_args", nargs=-1) # help: command: up @click.pass_context def up(ctx, extra_args): extra_args_list = list(extra_args) or None @@ -197,8 +278,10 @@ def up(ctx, extra_args): @command.command() -@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes") -@click.argument('extra_args', nargs=-1) # help: command: down +@click.option( + "--delete-volumes/--preserve-volumes", default=False, help="delete data volumes" +) +@click.argument("extra_args", nargs=-1) # help: command: down @click.pass_context def down(ctx, delete_volumes, extra_args): extra_args_list = list(extra_args) or None @@ -212,14 +295,14 @@ def ps(ctx): @command.command() -@click.argument('extra_args', nargs=-1) # help: command: port +@click.argument("extra_args", nargs=-1) # help: command: port @click.pass_context def port(ctx, extra_args): port_operation(ctx, extra_args) @command.command() -@click.argument('extra_args', nargs=-1) # help: command: exec +@click.argument("extra_args", nargs=-1) # help: command: exec @click.pass_context def exec(ctx, extra_args): exec_operation(ctx, extra_args) @@ -228,44 +311,49 @@ def exec(ctx, extra_args): @command.command() @click.option("--tail", "-n", default=None, help="number of lines to display") @click.option("--follow", "-f", is_flag=True, default=False, help="follow log output") -@click.argument('extra_args', nargs=-1) # help: command: logs +@click.argument("extra_args", nargs=-1) # help: command: logs @click.pass_context def logs(ctx, tail, follow, extra_args): logs_operation(ctx, tail, follow, extra_args) def get_stack_status(ctx, stack): - ctx_copy = copy.copy(ctx) ctx_copy.stack = stack cluster_context = _make_cluster_context(ctx_copy, stack, None, None, None, None) - deployer = Deployer(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster) + deployer = DockerDeployer( + type="compose", + deployment_context=None, + compose_files=cluster_context.compose_files, + compose_project_name=cluster_context.cluster, + compose_env_file=cluster_context.env_file, + ) # TODO: refactor to avoid duplicating this code above if ctx.verbose: print("Running compose ps") container_list = deployer.ps() - if len(container_list) > 0: - if ctx.debug: - print(f"Container list from compose ps: {container_list}") - return True - else: + if container_list is None or len(container_list) == 0: if ctx.debug: print("No containers found from compose ps") - False + return False + if ctx.debug: + print(f"Container list from compose ps: {container_list}") + return True def _make_runtime_env(ctx): container_exec_env = { "CERC_HOST_UID": f"{os.getuid()}", - "CERC_HOST_GID": f"{os.getgid()}" + "CERC_HOST_GID": f"{os.getgid()}", } container_exec_env.update({"CERC_SCRIPT_DEBUG": "true"} if ctx.debug else {}) return container_exec_env def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude): - # Create default unique, stable cluster name from confile file path and stack name if provided + # Create default unique, stable cluster name from confile file path and + # stack name if provided if deployment: path = os.path.realpath(os.path.abspath(compose_dir)) else: @@ -280,7 +368,8 @@ def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude) return cluster -# stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack +# stack has to be either PathLike pointing to a stack yml file, or a +# string with the name of a known stack def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): dev_root_path = get_dev_root_path(ctx) @@ -289,28 +378,37 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): if deployment: compose_dir = stack.joinpath("compose") else: - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose") + # See: + # https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + compose_dir = ( + Path(__file__).absolute().parent.parent.joinpath("data", "compose") + ) if cluster is None: - cluster = _make_default_cluster_name(deployment, compose_dir, stack, include, exclude) + cluster = _make_default_cluster_name( + deployment, compose_dir, stack, include, exclude + ) else: _make_default_cluster_name(deployment, compose_dir, stack, include, exclude) # See: https://stackoverflow.com/a/20885799/1701505 from stack_orchestrator import data + with resources.open_text(data, "pod-list.txt") as pod_list_file: all_pods = pod_list_file.read().splitlines() pods_in_scope = [] + cluster_config = None if stack: stack_config = get_parsed_stack_config(stack) - # TODO: syntax check the input here - pods_in_scope = stack_config['pods'] - cluster_config = stack_config['config'] if 'config' in stack_config else None + if stack_config is not None: + # TODO: syntax check the input here + pods_in_scope = stack_config["pods"] + cluster_config = ( + stack_config["config"] if "config" in stack_config else None + ) else: pods_in_scope = all_pods - cluster_config = None # Convert all pod definitions to v1.1 format pods_in_scope = _convert_to_new_format(pods_in_scope) @@ -330,29 +428,47 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): if include_exclude_check(pod_name, include, exclude): if pod_repository is None or pod_repository == "internal": if deployment: - compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml") + compose_file_name = os.path.join( + compose_dir, f"docker-compose-{pod_path}.yml" + ) else: compose_file_name = resolve_compose_file(stack, pod_name) else: if deployment: - compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml") + compose_file_name = os.path.join( + compose_dir, f"docker-compose-{pod_name}.yml" + ) pod_pre_start_command = pod.get("pre_start_command") pod_post_start_command = pod.get("post_start_command") - script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts") + script_dir = compose_dir.parent.joinpath( + "pods", pod_name, "scripts" + ) if pod_pre_start_command is not None: - pre_start_commands.append(os.path.join(script_dir, pod_pre_start_command)) + pre_start_commands.append( + os.path.join(script_dir, pod_pre_start_command) + ) if pod_post_start_command is not None: - post_start_commands.append(os.path.join(script_dir, pod_post_start_command)) + post_start_commands.append( + os.path.join(script_dir, pod_post_start_command) + ) else: # TODO: fix this code for external stack with scripts - pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"]) - compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml") + pod_root_dir = os.path.join( + dev_root_path, pod_repository.split("/")[-1], pod["path"] + ) + compose_file_name = os.path.join( + pod_root_dir, f"docker-compose-{pod_name}.yml" + ) pod_pre_start_command = pod.get("pre_start_command") pod_post_start_command = pod.get("post_start_command") if pod_pre_start_command is not None: - pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command)) + pre_start_commands.append( + os.path.join(pod_root_dir, pod_pre_start_command) + ) if pod_post_start_command is not None: - post_start_commands.append(os.path.join(pod_root_dir, pod_post_start_command)) + post_start_commands.append( + os.path.join(pod_root_dir, pod_post_start_command) + ) compose_files.append(compose_file_name) else: if ctx.verbose: @@ -361,7 +477,15 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): if ctx.verbose: print(f"files: {compose_files}") - return ClusterContext(ctx, cluster, compose_files, pre_start_commands, post_start_commands, cluster_config, env_file) + return ClusterContext( + ctx, + cluster, + compose_files, + pre_start_commands, + post_start_commands, + cluster_config, + env_file, + ) def _convert_to_new_format(old_pod_array): @@ -370,11 +494,7 @@ def _convert_to_new_format(old_pod_array): if isinstance(old_pod, dict): new_pod_array.append(old_pod) else: - new_pod = { - "name": old_pod, - "repository": "internal", - "path": old_pod - } + new_pod = {"name": old_pod, "repository": "internal", "path": old_pod} new_pod_array.append(new_pod) return new_pod_array @@ -388,14 +508,15 @@ def _run_command(ctx, cluster_name, command): command_env["CERC_SO_COMPOSE_PROJECT"] = cluster_name if ctx.debug: command_env["CERC_SCRIPT_DEBUG"] = "true" - command_result = subprocess.run(command_file, shell=True, env=command_env, cwd=command_dir) + command_result = subprocess.run( + command_file, shell=True, env=command_env, cwd=command_dir + ) if command_result.returncode != 0: print(f"FATAL Error running command: {command}") sys.exit(1) def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_env): - @dataclass class ConfigDirective: source_container: str @@ -413,24 +534,32 @@ def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_en container_config[directive].split(".")[0], container_config[directive].split(".")[1], container, - directive + directive, ) if ctx.verbose: - print(f"Setting {pd.destination_container}.{pd.destination_variable}" - f" = {pd.source_container}.{pd.source_variable}") + print( + f"Setting {pd.destination_container}.{pd.destination_variable}" + f" = {pd.source_container}.{pd.source_variable}" + ) # TODO: add a timeout waiting_for_data = True destination_output = "*** no output received yet ***" while waiting_for_data: - # TODO: fix the script paths so they're consistent between containers + # TODO: fix the script paths so they're consistent between + # containers source_value = None try: - source_value = deployer.execute(pd.source_container, - ["sh", "-c", - "sh /docker-entrypoint-scripts.d/export-" - f"{pd.source_variable}.sh"], - tty=False, - envs=container_exec_env) + source_value = deployer.execute( + pd.source_container, + [ + "sh", + "-c", + "sh /docker-entrypoint-scripts.d/export-" + f"{pd.source_variable}.sh", + ], + tty=False, + envs=container_exec_env, + ) except DeployerException as error: if ctx.debug: print(f"Docker exception reading config source: {error}") @@ -438,20 +567,28 @@ def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_en # "It returned with code 1" if "It returned with code 1" in str(error): if ctx.verbose: - print("Config export script returned an error, re-trying") - # If the script failed to execute (e.g. the file is not there) then we get: + print( + "Config export script returned an error, re-trying" + ) + # If the script failed to execute + # (e.g. the file is not there) then we get: # "It returned with code 2" if "It returned with code 2" in str(error): print(f"Fatal error reading config source: {error}") if source_value: if ctx.debug: print(f"fetched source value: {source_value}") - destination_output = deployer.execute(pd.destination_container, - ["sh", "-c", - f"sh /scripts/import-{pd.destination_variable}.sh" - f" {source_value}"], - tty=False, - envs=container_exec_env) + destination_output = deployer.execute( + pd.destination_container, + [ + "sh", + "-c", + f"sh /scripts/import-{pd.destination_variable}.sh" + f" {source_value}", + ], + tty=False, + envs=container_exec_env, + ) waiting_for_data = False if ctx.debug and not waiting_for_data: print(f"destination output: {destination_output}") @@ -460,3 +597,4 @@ def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_en command.add_command(deployment_init) command.add_command(deployment_create) command.add_command(deployment_setup) +command.add_command(k8s_command.command, "k8s") diff --git a/stack_orchestrator/deploy/deploy_types.py b/stack_orchestrator/deploy/deploy_types.py index f59d9f67..202e0fa5 100644 --- a/stack_orchestrator/deploy/deploy_types.py +++ b/stack_orchestrator/deploy/deploy_types.py @@ -13,7 +13,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import List, Mapping +from typing import List, Mapping, Optional from dataclasses import dataclass from stack_orchestrator.command_types import CommandOptions from stack_orchestrator.deploy.deployer import Deployer @@ -21,20 +21,21 @@ from stack_orchestrator.deploy.deployer import Deployer @dataclass class ClusterContext: - options: CommandOptions # TODO: this should be in its own object not stuffed in here - cluster: str + # TODO: this should be in its own object not stuffed in here + options: CommandOptions + cluster: Optional[str] compose_files: List[str] pre_start_commands: List[str] post_start_commands: List[str] - config: str - env_file: str + config: Optional[str] + env_file: Optional[str] @dataclass class DeployCommandContext: stack: str cluster_context: ClusterContext - deployer: Deployer + deployer: Optional[Deployer] @dataclass diff --git a/stack_orchestrator/deploy/deploy_util.py b/stack_orchestrator/deploy/deploy_util.py index 9e204baa..65111653 100644 --- a/stack_orchestrator/deploy/deploy_util.py +++ b/stack_orchestrator/deploy/deploy_util.py @@ -15,7 +15,12 @@ from typing import List, Any from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping -from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_pod_list, resolve_compose_file +from stack_orchestrator.util import ( + get_parsed_stack_config, + get_yaml, + get_pod_list, + resolve_compose_file, +) from stack_orchestrator.opts import opts @@ -38,7 +43,7 @@ def _container_image_from_service(stack: str, service: str): def parsed_pod_files_map_from_file_names(pod_files): - parsed_pod_yaml_map : Any = {} + parsed_pod_yaml_map: Any = {} for pod_file in pod_files: with open(pod_file, "r") as pod_file_descriptor: parsed_pod_file = get_yaml().load(pod_file_descriptor) @@ -73,19 +78,28 @@ def _volumes_to_docker(mounts: List[VolumeMapping]): return result -def run_container_command(ctx: DeployCommandContext, service: str, command: str, mounts: List[VolumeMapping]): +def run_container_command( + ctx: DeployCommandContext, service: str, command: str, mounts: List[VolumeMapping] +): deployer = ctx.deployer + if deployer is None: + raise ValueError("Deployer is not configured") container_image = _container_image_from_service(ctx.stack, service) + if container_image is None: + raise ValueError(f"Container image not found for service: {service}") docker_volumes = _volumes_to_docker(mounts) if ctx.cluster_context.options.debug: print(f"Running this command in {service} container: {command}") docker_output = deployer.run( container_image, - ["-c", command], entrypoint="sh", - # Current laconicd container has a bug where it crashes when run not as root - # Commented out line below is a workaround. Created files end up owned by root on the host + ["-c", command], + entrypoint="sh", + # Current laconicd container has a bug where it crashes when run not + # as root + # Commented out line below is a workaround. Created files end up + # owned by root on the host # user=f"{os.getuid()}:{os.getgid()}", - volumes=docker_volumes - ) + volumes=docker_volumes, + ) # There doesn't seem to be a way to get an exit code from docker.run() return (docker_output, 0) diff --git a/stack_orchestrator/deploy/deployer.py b/stack_orchestrator/deploy/deployer.py index 15db44c2..d8fb656b 100644 --- a/stack_orchestrator/deploy/deployer.py +++ b/stack_orchestrator/deploy/deployer.py @@ -15,10 +15,10 @@ from abc import ABC, abstractmethod from pathlib import Path +from typing import Optional class Deployer(ABC): - @abstractmethod def up(self, detach, skip_cluster_management, services): pass @@ -52,7 +52,21 @@ class Deployer(ABC): pass @abstractmethod - def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): + def run( + self, + image: str, + command=None, + user=None, + volumes=None, + entrypoint=None, + env={}, + ports=[], + detach=False, + ): + pass + + @abstractmethod + def run_job(self, job_name: str, release_name: Optional[str] = None): pass @@ -62,7 +76,6 @@ class DeployerException(Exception): class DeployerConfigGenerator(ABC): - @abstractmethod def generate(self, deployment_dir: Path): pass diff --git a/stack_orchestrator/deploy/deployer_factory.py b/stack_orchestrator/deploy/deployer_factory.py index 2d01729e..1de14cc5 100644 --- a/stack_orchestrator/deploy/deployer_factory.py +++ b/stack_orchestrator/deploy/deployer_factory.py @@ -14,8 +14,14 @@ # along with this program. If not, see . from stack_orchestrator import constants -from stack_orchestrator.deploy.k8s.deploy_k8s import K8sDeployer, K8sDeployerConfigGenerator -from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer, DockerDeployerConfigGenerator +from stack_orchestrator.deploy.k8s.deploy_k8s import ( + K8sDeployer, + K8sDeployerConfigGenerator, +) +from stack_orchestrator.deploy.compose.deploy_docker import ( + DockerDeployer, + DockerDeployerConfigGenerator, +) def getDeployerConfigGenerator(type: str, deployment_context): @@ -27,10 +33,27 @@ def getDeployerConfigGenerator(type: str, deployment_context): print(f"ERROR: deploy-to {type} is not valid") -def getDeployer(type: str, deployment_context, compose_files, compose_project_name, compose_env_file): +def getDeployer( + type: str, deployment_context, compose_files, compose_project_name, compose_env_file +): if type == "compose" or type is None: - return DockerDeployer(type, deployment_context, compose_files, compose_project_name, compose_env_file) - elif type == type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type: - return K8sDeployer(type, deployment_context, compose_files, compose_project_name, compose_env_file) + return DockerDeployer( + type, + deployment_context, + compose_files, + compose_project_name, + compose_env_file, + ) + elif ( + type == type == constants.k8s_deploy_type + or type == constants.k8s_kind_deploy_type + ): + return K8sDeployer( + type, + deployment_context, + compose_files, + compose_project_name, + compose_env_file, + ) else: print(f"ERROR: deploy-to {type} is not valid") diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index 7021c733..35abea3c 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -18,8 +18,19 @@ from pathlib import Path import sys from stack_orchestrator import constants from stack_orchestrator.deploy.images import push_images_operation -from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation, status_operation -from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context, update_operation +from stack_orchestrator.deploy.deploy import ( + up_operation, + down_operation, + ps_operation, + port_operation, + status_operation, +) +from stack_orchestrator.deploy.deploy import ( + exec_operation, + logs_operation, + create_deploy_context, + update_operation, +) from stack_orchestrator.deploy.deploy_types import DeployCommandContext from stack_orchestrator.deploy.deployment_context import DeploymentContext @@ -28,7 +39,7 @@ from stack_orchestrator.deploy.deployment_context import DeploymentContext @click.option("--dir", required=True, help="path to deployment directory") @click.pass_context def command(ctx, dir): - '''manage a deployment''' + """manage a deployment""" # Check that --stack wasn't supplied if ctx.parent.obj.stack: @@ -40,7 +51,10 @@ def command(ctx, dir): print(f"Error: deployment directory {dir} does not exist") sys.exit(1) if not dir_path.is_dir(): - print(f"Error: supplied deployment directory path {dir} exists but is a file not a directory") + print( + f"Error: supplied deployment directory path {dir} exists but is a " + "file not a directory" + ) sys.exit(1) # Store the deployment context for subcommands deployment_context = DeploymentContext() @@ -57,16 +71,31 @@ def make_deploy_context(ctx) -> DeployCommandContext: else: deployment_type = constants.compose_deploy_type stack = context.deployment_dir - return create_deploy_context(ctx.parent.parent.obj, context, stack, None, None, - cluster_name, env_file, deployment_type) + return create_deploy_context( + ctx.parent.parent.obj, + context, + stack, + None, + None, + cluster_name, + env_file, + deployment_type, + ) # TODO: remove legacy up command since it's an alias for start @command.command() -@click.option("--stay-attached/--detatch-terminal", default=False, help="detatch or not to see container stdout") -@click.option("--skip-cluster-management/--perform-cluster-management", - default=False, help="Skip cluster initialization/tear-down (only for kind-k8s deployments)") -@click.argument('extra_args', nargs=-1) # help: command: up +@click.option( + "--stay-attached/--detatch-terminal", + default=False, + help="detatch or not to see container stdout", +) +@click.option( + "--skip-cluster-management/--perform-cluster-management", + default=False, + help="Skip cluster initialization/tear-down (only for kind-k8s deployments)", +) +@click.argument("extra_args", nargs=-1) # help: command: up @click.pass_context def up(ctx, stay_attached, skip_cluster_management, extra_args): ctx.obj = make_deploy_context(ctx) @@ -76,10 +105,17 @@ def up(ctx, stay_attached, skip_cluster_management, extra_args): # start is the preferred alias for up @command.command() -@click.option("--stay-attached/--detatch-terminal", default=False, help="detatch or not to see container stdout") -@click.option("--skip-cluster-management/--perform-cluster-management", - default=False, help="Skip cluster initialization/tear-down (only for kind-k8s deployments)") -@click.argument('extra_args', nargs=-1) # help: command: up +@click.option( + "--stay-attached/--detatch-terminal", + default=False, + help="detatch or not to see container stdout", +) +@click.option( + "--skip-cluster-management/--perform-cluster-management", + default=False, + help="Skip cluster initialization/tear-down (only for kind-k8s deployments)", +) +@click.argument("extra_args", nargs=-1) # help: command: up @click.pass_context def start(ctx, stay_attached, skip_cluster_management, extra_args): ctx.obj = make_deploy_context(ctx) @@ -89,10 +125,15 @@ def start(ctx, stay_attached, skip_cluster_management, extra_args): # TODO: remove legacy up command since it's an alias for stop @command.command() -@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes") -@click.option("--skip-cluster-management/--perform-cluster-management", - default=False, help="Skip cluster initialization/tear-down (only for kind-k8s deployments)") -@click.argument('extra_args', nargs=-1) # help: command: down +@click.option( + "--delete-volumes/--preserve-volumes", default=False, help="delete data volumes" +) +@click.option( + "--skip-cluster-management/--perform-cluster-management", + default=False, + help="Skip cluster initialization/tear-down (only for kind-k8s deployments)", +) +@click.argument("extra_args", nargs=-1) # help: command: down @click.pass_context def down(ctx, delete_volumes, skip_cluster_management, extra_args): # Get the stack config file name @@ -103,10 +144,15 @@ def down(ctx, delete_volumes, skip_cluster_management, extra_args): # stop is the preferred alias for down @command.command() -@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes") -@click.option("--skip-cluster-management/--perform-cluster-management", - default=False, help="Skip cluster initialization/tear-down (only for kind-k8s deployments)") -@click.argument('extra_args', nargs=-1) # help: command: down +@click.option( + "--delete-volumes/--preserve-volumes", default=False, help="delete data volumes" +) +@click.option( + "--skip-cluster-management/--perform-cluster-management", + default=False, + help="Skip cluster initialization/tear-down (only for kind-k8s deployments)", +) +@click.argument("extra_args", nargs=-1) # help: command: down @click.pass_context def stop(ctx, delete_volumes, skip_cluster_management, extra_args): # TODO: add cluster name and env file here @@ -130,7 +176,7 @@ def push_images(ctx): @command.command() -@click.argument('extra_args', nargs=-1) # help: command: port +@click.argument("extra_args", nargs=-1) # help: command: port @click.pass_context def port(ctx, extra_args): ctx.obj = make_deploy_context(ctx) @@ -138,7 +184,7 @@ def port(ctx, extra_args): @command.command() -@click.argument('extra_args', nargs=-1) # help: command: exec +@click.argument("extra_args", nargs=-1) # help: command: exec @click.pass_context def exec(ctx, extra_args): ctx.obj = make_deploy_context(ctx) @@ -148,7 +194,7 @@ def exec(ctx, extra_args): @command.command() @click.option("--tail", "-n", default=None, help="number of lines to display") @click.option("--follow", "-f", is_flag=True, default=False, help="follow log output") -@click.argument('extra_args', nargs=-1) # help: command: logs +@click.argument("extra_args", nargs=-1) # help: command: logs @click.pass_context def logs(ctx, tail, follow, extra_args): ctx.obj = make_deploy_context(ctx) @@ -167,3 +213,18 @@ def status(ctx): def update(ctx): ctx.obj = make_deploy_context(ctx) update_operation(ctx) + + +@command.command() +@click.argument("job_name") +@click.option( + "--helm-release", + help="Helm release name (for k8s helm chart deployments, defaults to chart name)", +) +@click.pass_context +def run_job(ctx, job_name, helm_release): + """run a one-time job from the stack""" + from stack_orchestrator.deploy.deploy import run_job_operation + + ctx.obj = make_deploy_context(ctx) + run_job_operation(ctx, job_name, helm_release) diff --git a/stack_orchestrator/deploy/deployment_context.py b/stack_orchestrator/deploy/deployment_context.py index 2ffc5b89..79fc4bb9 100644 --- a/stack_orchestrator/deploy/deployment_context.py +++ b/stack_orchestrator/deploy/deployment_context.py @@ -1,4 +1,3 @@ - # Copyright © 2022, 2023 Vulcanize # This program is free software: you can redistribute it and/or modify @@ -71,17 +70,15 @@ class DeploymentContext: self.id = f"{constants.cluster_name_prefix}{hash}" def modify_yaml(self, file_path: Path, modifier_func): - """ - Load a YAML from the deployment, apply a modification function, and write it back. - """ + """Load a YAML, apply a modification function, and write it back.""" if not file_path.absolute().is_relative_to(self.deployment_dir): raise ValueError(f"File is not inside deployment directory: {file_path}") yaml = get_yaml() - with open(file_path, 'r') as f: + with open(file_path, "r") as f: yaml_data = yaml.load(f) modifier_func(yaml_data) - with open(file_path, 'w') as f: + with open(file_path, "w") as f: yaml.dump(yaml_data, f) diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 5daffad8..601f6c77 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -24,10 +24,23 @@ from secrets import token_hex import sys from stack_orchestrator import constants from stack_orchestrator.opts import opts -from stack_orchestrator.util import (get_stack_path, get_parsed_deployment_spec, get_parsed_stack_config, - global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts, - get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file, - resolve_config_dir) +from stack_orchestrator.util import ( + get_stack_path, + get_parsed_deployment_spec, + get_parsed_stack_config, + global_options, + get_yaml, + get_pod_list, + get_pod_file_path, + pod_has_scripts, + get_pod_script_paths, + get_plugin_code_paths, + error_exit, + env_var_map_from_file, + resolve_config_dir, + get_job_list, + get_job_file_path, +) from stack_orchestrator.deploy.spec import Spec from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator @@ -45,21 +58,21 @@ def _get_ports(stack): yaml = get_yaml() for pod in pods: pod_file_path = get_pod_file_path(stack, parsed_stack, pod) + if pod_file_path is None: + continue parsed_pod_file = yaml.load(open(pod_file_path, "r")) if "services" in parsed_pod_file: for svc_name, svc in parsed_pod_file["services"].items(): if "ports" in svc: - # Ports can appear as strings or numbers. We normalize them as strings. + # Ports can appear as strings or numbers. We normalize them as + # strings. ports[svc_name] = [str(x) for x in svc["ports"]] return ports def _get_named_volumes(stack): # Parse the compose files looking for named volumes - named_volumes = { - "rw": [], - "ro": [] - } + named_volumes = {"rw": [], "ro": []} parsed_stack = get_parsed_stack_config(stack) pods = get_pod_list(parsed_stack) yaml = get_yaml() @@ -75,12 +88,14 @@ def _get_named_volumes(stack): ret[svc_name] = { "volume": parts[0], "mount": parts[1], - "options": parts[2] if len(parts) == 3 else None + "options": parts[2] if len(parts) == 3 else None, } return ret for pod in pods: pod_file_path = get_pod_file_path(stack, parsed_stack, pod) + if pod_file_path is None: + continue parsed_pod_file = yaml.load(open(pod_file_path, "r")) if "volumes" in parsed_pod_file: volumes = parsed_pod_file["volumes"] @@ -88,7 +103,10 @@ def _get_named_volumes(stack): for vu in find_vol_usage(parsed_pod_file, volume).values(): read_only = vu["options"] == "ro" if read_only: - if vu["volume"] not in named_volumes["rw"] and vu["volume"] not in named_volumes["ro"]: + if ( + vu["volume"] not in named_volumes["rw"] + and vu["volume"] not in named_volumes["ro"] + ): named_volumes["ro"].append(vu["volume"]) else: if vu["volume"] not in named_volumes["rw"]: @@ -108,10 +126,13 @@ def _create_bind_dir_if_relative(volume, path_string, compose_dir): absolute_path.mkdir(parents=True, exist_ok=True) else: if not path.exists(): - print(f"WARNING: mount path for volume {volume} does not exist: {path_string}") + print( + f"WARNING: mount path for volume {volume} does not exist: {path_string}" + ) -# See: https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml +# See: +# https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml def _fixup_pod_file(pod, spec, compose_dir): deployment_type = spec[constants.deploy_to_key] # Fix up volumes @@ -123,7 +144,11 @@ def _fixup_pod_file(pod, spec, compose_dir): if volume in spec_volumes: volume_spec = spec_volumes[volume] if volume_spec: - volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}" + volume_spec_fixedup = ( + volume_spec + if Path(volume_spec).is_absolute() + else f".{volume_spec}" + ) _create_bind_dir_if_relative(volume, volume_spec, compose_dir) # this is Docker specific if spec.is_docker_deployment(): @@ -132,8 +157,8 @@ def _fixup_pod_file(pod, spec, compose_dir): "driver_opts": { "type": "none", "device": volume_spec_fixedup, - "o": "bind" - } + "o": "bind", + }, } pod["volumes"][volume] = new_volume_spec @@ -181,6 +206,8 @@ def call_stack_deploy_init(deploy_command_context): for python_file_path in python_file_paths: if python_file_path.exists(): spec = util.spec_from_file_location("commands", python_file_path) + if spec is None or spec.loader is None: + continue imported_stack = util.module_from_spec(spec) spec.loader.exec_module(imported_stack) if _has_method(imported_stack, "init"): @@ -189,12 +216,17 @@ def call_stack_deploy_init(deploy_command_context): init_done = True else: # TODO: remove this restriction - print(f"Skipping init() from plugin {python_file_path}. Only one init() is allowed.") + print( + f"Skipping init() from plugin {python_file_path}. " + "Only one init() is allowed." + ) return ret # TODO: fold this with function above -def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetupCommand, extra_args): +def call_stack_deploy_setup( + deploy_command_context, parameters: LaconicStackSetupCommand, extra_args +): # Link with the python file in the stack # Call a function in it # If no function found, return None @@ -202,6 +234,8 @@ def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetu for python_file_path in python_file_paths: if python_file_path.exists(): spec = util.spec_from_file_location("commands", python_file_path) + if spec is None or spec.loader is None: + continue imported_stack = util.module_from_spec(spec) spec.loader.exec_module(imported_stack) if _has_method(imported_stack, "setup"): @@ -217,6 +251,8 @@ def call_stack_deploy_create(deployment_context, extra_args): for python_file_path in python_file_paths: if python_file_path.exists(): spec = util.spec_from_file_location("commands", python_file_path) + if spec is None or spec.loader is None: + continue imported_stack = util.module_from_spec(spec) spec.loader.exec_module(imported_stack) if _has_method(imported_stack, "create"): @@ -247,7 +283,13 @@ def _find_extra_config_dirs(parsed_pod_file, pod): def _get_mapped_ports(stack: str, map_recipe: str): - port_map_recipes = ["any-variable-random", "localhost-same", "any-same", "localhost-fixed-random", "any-fixed-random"] + port_map_recipes = [ + "any-variable-random", + "localhost-same", + "any-same", + "localhost-fixed-random", + "any-fixed-random", + ] ports = _get_ports(stack) if ports: # Implement any requested mapping recipe @@ -259,7 +301,9 @@ def _get_mapped_ports(stack: str, map_recipe: str): orig_port = ports_array[x] # Strip /udp suffix if present bare_orig_port = orig_port.replace("/udp", "") - random_port = random.randint(20000, 50000) # Beware: we're relying on luck to not collide + random_port = random.randint( + 20000, 50000 + ) # Beware: we're relying on luck to not collide if map_recipe == "any-variable-random": # This is the default so take no action pass @@ -278,7 +322,10 @@ def _get_mapped_ports(stack: str, map_recipe: str): else: print("Error: bad map_recipe") else: - print(f"Error: --map-ports-to-host must specify one of: {port_map_recipes}") + print( + f"Error: --map-ports-to-host must specify one of: " + f"{port_map_recipes}" + ) sys.exit(1) return ports @@ -303,33 +350,54 @@ def _parse_config_variables(variable_values: str): @click.command() @click.option("--config", help="Provide config variables for the deployment") -@click.option("--config-file", help="Provide config variables in a file for the deployment") +@click.option( + "--config-file", help="Provide config variables in a file for the deployment" +) @click.option("--kube-config", help="Provide a config file for a k8s deployment") -@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") +@click.option( + "--image-registry", + help="Provide a container image registry url for this k8s cluster", +) @click.option("--output", required=True, help="Write yaml spec file here") -@click.option("--map-ports-to-host", required=False, - help="Map ports to the host as one of: any-variable-random (default), " - "localhost-same, any-same, localhost-fixed-random, any-fixed-random") +@click.option( + "--map-ports-to-host", + required=False, + help="Map ports to the host as one of: any-variable-random (default), " + "localhost-same, any-same, localhost-fixed-random, any-fixed-random", +) @click.pass_context -def init(ctx, config, config_file, kube_config, image_registry, output, map_ports_to_host): +def init( + ctx, config, config_file, kube_config, image_registry, output, map_ports_to_host +): stack = global_options(ctx).stack deployer_type = ctx.obj.deployer.type deploy_command_context = ctx.obj return init_operation( deploy_command_context, - stack, deployer_type, - config, config_file, + stack, + deployer_type, + config, + config_file, kube_config, image_registry, output, - map_ports_to_host) + map_ports_to_host, + ) # The init command's implementation is in a separate function so that we can # call it from other commands, bypassing the click decoration stuff -def init_operation(deploy_command_context, stack, deployer_type, config, - config_file, kube_config, image_registry, output, map_ports_to_host): - +def init_operation( + deploy_command_context, + stack, + deployer_type, + config, + config_file, + kube_config, + image_registry, + output, + map_ports_to_host, +): default_spec_file_content = call_stack_deploy_init(deploy_command_context) spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type} if deployer_type == "k8s": @@ -340,13 +408,20 @@ def init_operation(deploy_command_context, stack, deployer_type, config, if image_registry: spec_file_content.update({constants.image_registry_key: image_registry}) else: - print("WARNING: --image-registry not specified, only default container registries (eg, Docker Hub) will be available") + print( + "WARNING: --image-registry not specified, only default container " + "registries (eg, Docker Hub) will be available" + ) else: # Check for --kube-config supplied for non-relevant deployer types if kube_config is not None: - error_exit(f"--kube-config is not allowed with a {deployer_type} deployment") + error_exit( + f"--kube-config is not allowed with a {deployer_type} deployment" + ) if image_registry is not None: - error_exit(f"--image-registry is not allowed with a {deployer_type} deployment") + error_exit( + f"--image-registry is not allowed with a {deployer_type} deployment" + ) if default_spec_file_content: spec_file_content.update(default_spec_file_content) config_variables = _parse_config_variables(config) @@ -368,7 +443,9 @@ def init_operation(deploy_command_context, stack, deployer_type, config, spec_file_content.update({"config": merged_config}) ports = _get_mapped_ports(stack, map_ports_to_host) - spec_file_content.update({"network": {"ports": ports}}) + orig_network = spec_file_content.get("network", {}) + orig_network["ports"] = ports + spec_file_content["network"] = orig_network named_volumes = _get_named_volumes(stack) if named_volumes: @@ -393,7 +470,9 @@ def init_operation(deploy_command_context, stack, deployer_type, config, spec_file_content["configmaps"] = configmap_descriptors if opts.o.debug: - print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") + print( + f"Creating spec file for stack: {stack} with content: {spec_file_content}" + ) with open(output, "w") as output_file: get_yaml().dump(spec_file_content, output_file) @@ -441,22 +520,54 @@ def _check_volume_definitions(spec): @click.command() -@click.option("--spec-file", required=True, help="Spec file to use to create this deployment") +@click.option( + "--spec-file", required=True, help="Spec file to use to create this deployment" +) @click.option("--deployment-dir", help="Create deployment files in this directory") -@click.argument('extra_args', nargs=-1, type=click.UNPROCESSED) +@click.option( + "--helm-chart", + is_flag=True, + default=False, + help="Generate Helm chart instead of deploying (k8s only)", +) +# TODO: Hack +@click.option("--network-dir", help="Network configuration supplied in this directory") +@click.option("--initial-peers", help="Initial set of persistent peers") +@click.argument("extra_args", nargs=-1, type=click.UNPROCESSED) @click.pass_context -def create(ctx, spec_file, deployment_dir, extra_args): +def create( + ctx, spec_file, deployment_dir, helm_chart, network_dir, initial_peers, extra_args +): deployment_command_context = ctx.obj - return create_operation(deployment_command_context, spec_file, deployment_dir, extra_args) + return create_operation( + deployment_command_context, + spec_file, + deployment_dir, + helm_chart, + network_dir, + initial_peers, + extra_args, + ) # The init command's implementation is in a separate function so that we can # call it from other commands, bypassing the click decoration stuff -def create_operation(deployment_command_context, spec_file, deployment_dir, extra_args): - parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file)) +def create_operation( + deployment_command_context, + spec_file, + deployment_dir, + helm_chart=False, + network_dir=None, + initial_peers=None, + extra_args=(), +): + parsed_spec = Spec( + os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file) + ) _check_volume_definitions(parsed_spec) stack_name = parsed_spec["stack"] deployment_type = parsed_spec[constants.deploy_to_key] + stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name) parsed_stack = get_parsed_stack_config(stack_name) if opts.o.debug: @@ -471,13 +582,30 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, extr # Copy spec file and the stack file into the deployment dir copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name)) copyfile(stack_file, deployment_dir_path.joinpath(constants.stack_file_name)) + + # Create deployment.yml with cluster-id _create_deployment_file(deployment_dir_path) + + # Branch to Helm chart generation flow if --helm-chart flag is set + if deployment_type == "k8s" and helm_chart: + from stack_orchestrator.deploy.k8s.helm.chart_generator import ( + generate_helm_chart, + ) + + generate_helm_chart(stack_name, spec_file, deployment_dir_path) + return # Exit early for helm chart generation + + # Existing deployment flow continues unchanged # Copy any config varibles from the spec file into an env file suitable for compose - _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name)) + _write_config_file( + spec_file, deployment_dir_path.joinpath(constants.config_file_name) + ) # Copy any k8s config file into the deployment dir if deployment_type == "k8s": - _write_kube_config_file(Path(parsed_spec[constants.kube_config_key]), - deployment_dir_path.joinpath(constants.kube_config_filename)) + _write_kube_config_file( + Path(parsed_spec[constants.kube_config_key]), + deployment_dir_path.joinpath(constants.kube_config_filename), + ) # Copy the pod files into the deployment dir, fixing up content pods = get_pod_list(parsed_stack) destination_compose_dir = deployment_dir_path.joinpath("compose") @@ -487,6 +615,8 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, extr yaml = get_yaml() for pod in pods: pod_file_path = get_pod_file_path(stack_name, parsed_stack, pod) + if pod_file_path is None: + continue parsed_pod_file = yaml.load(open(pod_file_path, "r")) extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod) destination_pod_dir = destination_pods_dir.joinpath(pod) @@ -494,7 +624,9 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, extr if opts.o.debug: print(f"extra config dirs: {extra_config_dirs}") _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir) - with open(destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w") as output_file: + with open( + destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w" + ) as output_file: yaml.dump(parsed_pod_file, output_file) # Copy the config files for the pod, if any config_dirs = {pod} @@ -502,8 +634,11 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, extr for config_dir in config_dirs: source_config_dir = resolve_config_dir(stack_name, config_dir) if os.path.exists(source_config_dir): - destination_config_dir = deployment_dir_path.joinpath("config", config_dir) - # If the same config dir appears in multiple pods, it may already have been copied + destination_config_dir = deployment_dir_path.joinpath( + "config", config_dir + ) + # If the same config dir appears in multiple pods, it may already have + # been copied if not os.path.exists(destination_config_dir): copytree(source_config_dir, destination_config_dir) # Copy the script files for the pod, if any @@ -516,8 +651,12 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, extr for configmap in parsed_spec.get_configmaps(): source_config_dir = resolve_config_dir(stack_name, configmap) if os.path.exists(source_config_dir): - destination_config_dir = deployment_dir_path.joinpath("configmaps", configmap) - copytree(source_config_dir, destination_config_dir, dirs_exist_ok=True) + destination_config_dir = deployment_dir_path.joinpath( + "configmaps", configmap + ) + copytree( + source_config_dir, destination_config_dir, dirs_exist_ok=True + ) else: # TODO: We should probably only do this if the volume is marked :ro. for volume_name, volume_path in parsed_spec.get_volumes().items(): @@ -526,20 +665,51 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, extr if os.path.exists(source_config_dir) and os.listdir(source_config_dir): destination_config_dir = deployment_dir_path.joinpath(volume_path) # Only copy if the destination exists and _is_ empty. - if os.path.exists(destination_config_dir) and not os.listdir(destination_config_dir): - copytree(source_config_dir, destination_config_dir, dirs_exist_ok=True) + if os.path.exists(destination_config_dir) and not os.listdir( + destination_config_dir + ): + copytree( + source_config_dir, + destination_config_dir, + dirs_exist_ok=True, + ) + + # Copy the job files into the deployment dir (for Docker deployments) + jobs = get_job_list(parsed_stack) + if jobs and not parsed_spec.is_kubernetes_deployment(): + destination_compose_jobs_dir = deployment_dir_path.joinpath("compose-jobs") + os.mkdir(destination_compose_jobs_dir) + for job in jobs: + job_file_path = get_job_file_path(stack_name, parsed_stack, job) + if job_file_path and job_file_path.exists(): + parsed_job_file = yaml.load(open(job_file_path, "r")) + _fixup_pod_file(parsed_job_file, parsed_spec, destination_compose_dir) + with open( + destination_compose_jobs_dir.joinpath( + "docker-compose-%s.yml" % job + ), + "w", + ) as output_file: + yaml.dump(parsed_job_file, output_file) + if opts.o.debug: + print(f"Copied job compose file: {job}") # Delegate to the stack's Python code - # The deploy create command doesn't require a --stack argument so we need to insert the - # stack member here. + # The deploy create command doesn't require a --stack argument so we need + # to insert the stack member here. deployment_command_context.stack = stack_name deployment_context = DeploymentContext() deployment_context.init(deployment_dir_path) # Call the deployer to generate any deployer-specific files (e.g. for kind) - deployer_config_generator = getDeployerConfigGenerator(deployment_type, deployment_context) + deployer_config_generator = getDeployerConfigGenerator( + deployment_type, deployment_context + ) # TODO: make deployment_dir_path a Path above - deployer_config_generator.generate(deployment_dir_path) - call_stack_deploy_create(deployment_context, extra_args) + if deployer_config_generator is not None: + deployer_config_generator.generate(deployment_dir_path) + call_stack_deploy_create( + deployment_context, [network_dir, initial_peers, *extra_args] + ) # TODO: this code should be in the stack .py files but @@ -549,18 +719,50 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, extr @click.option("--node-moniker", help="Moniker for this node") @click.option("--chain-id", help="The new chain id") @click.option("--key-name", help="Name for new node key") -@click.option("--gentx-files", help="List of comma-delimited gentx filenames from other nodes") -@click.option("--gentx-addresses", type=str, help="List of comma-delimited validator addresses for other nodes") +@click.option( + "--gentx-files", help="List of comma-delimited gentx filenames from other nodes" +) +@click.option( + "--gentx-addresses", + type=str, + help="List of comma-delimited validator addresses for other nodes", +) @click.option("--genesis-file", help="Genesis file for the network") -@click.option("--initialize-network", is_flag=True, default=False, help="Initialize phase") +@click.option( + "--initialize-network", is_flag=True, default=False, help="Initialize phase" +) @click.option("--join-network", is_flag=True, default=False, help="Join phase") @click.option("--connect-network", is_flag=True, default=False, help="Connect phase") @click.option("--create-network", is_flag=True, default=False, help="Create phase") @click.option("--network-dir", help="Directory for network files") -@click.argument('extra_args', nargs=-1) +@click.argument("extra_args", nargs=-1) @click.pass_context -def setup(ctx, node_moniker, chain_id, key_name, gentx_files, gentx_addresses, genesis_file, initialize_network, join_network, - connect_network, create_network, network_dir, extra_args): - parmeters = LaconicStackSetupCommand(chain_id, node_moniker, key_name, initialize_network, join_network, connect_network, - create_network, gentx_files, gentx_addresses, genesis_file, network_dir) +def setup( + ctx, + node_moniker, + chain_id, + key_name, + gentx_files, + gentx_addresses, + genesis_file, + initialize_network, + join_network, + connect_network, + create_network, + network_dir, + extra_args, +): + parmeters = LaconicStackSetupCommand( + chain_id, + node_moniker, + key_name, + initialize_network, + join_network, + connect_network, + create_network, + gentx_files, + gentx_addresses, + genesis_file, + network_dir, + ) call_stack_deploy_setup(ctx.obj, parmeters, extra_args) diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index f2af1c09..2c57bf47 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -32,7 +32,9 @@ def _image_needs_pushed(image: str): def _remote_tag_for_image(image: str, remote_repo_url: str): # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy major_parts = image.split("/", 2) - image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0] + image_name_with_version = ( + major_parts[1] if 2 == len(major_parts) else major_parts[0] + ) (image_name, image_version) = image_name_with_version.split(":") if image_version == "local": return f"{remote_repo_url}/{image_name}:deploy" @@ -61,17 +63,22 @@ def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags): docker = DockerClient() remote_tag = _remote_tag_for_image(local_tag, remote_repo_url) - new_remote_tags = [_remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags] + new_remote_tags = [ + _remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags + ] docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags) def remote_tag_for_image_unique(image: str, remote_repo_url: str, deployment_id: str): # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy major_parts = image.split("/", 2) - image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0] + image_name_with_version = ( + major_parts[1] if 2 == len(major_parts) else major_parts[0] + ) (image_name, image_version) = image_name_with_version.split(":") if image_version == "local": - # Salt the tag with part of the deployment id to make it unique to this deployment + # Salt the tag with part of the deployment id to make it unique to this + # deployment deployment_tag = deployment_id[-8:] return f"{remote_repo_url}/{image_name}:deploy-{deployment_tag}" else: @@ -79,7 +86,9 @@ def remote_tag_for_image_unique(image: str, remote_repo_url: str, deployment_id: # TODO: needs lots of error handling -def push_images_operation(command_context: DeployCommandContext, deployment_context: DeploymentContext): +def push_images_operation( + command_context: DeployCommandContext, deployment_context: DeploymentContext +): # Get the list of images for the stack cluster_context = command_context.cluster_context images: Set[str] = images_for_deployment(cluster_context.compose_files) @@ -88,14 +97,18 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont docker = DockerClient() for image in images: if _image_needs_pushed(image): - remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id) + remote_tag = remote_tag_for_image_unique( + image, remote_repo_url, deployment_context.id + ) if opts.o.verbose: print(f"Tagging {image} to {remote_tag}") docker.image.tag(image, remote_tag) # Run docker push commands to upload for image in images: if _image_needs_pushed(image): - remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id) + remote_tag = remote_tag_for_image_unique( + image, remote_repo_url, deployment_context.id + ) if opts.o.verbose: print(f"Pushing image {remote_tag}") docker.image.push(remote_tag) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index be1b2e3d..42c41b4b 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -17,30 +17,41 @@ import os import base64 from kubernetes import client -from typing import Any, List, Set +from typing import Any, List, Optional, Set from stack_orchestrator.opts import opts from stack_orchestrator.util import env_var_map_from_file -from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files +from stack_orchestrator.deploy.k8s.helpers import ( + named_volumes_from_pod_files, + volume_mounts_for_service, + volumes_for_pod_files, +) from stack_orchestrator.deploy.k8s.helpers import get_kind_pv_bind_mount_path -from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map, envs_from_compose_file, merge_envs -from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment +from stack_orchestrator.deploy.k8s.helpers import ( + envs_from_environment_variables_map, + envs_from_compose_file, + merge_envs, +) +from stack_orchestrator.deploy.deploy_util import ( + parsed_pod_files_map_from_file_names, + images_for_deployment, +) from stack_orchestrator.deploy.deploy_types import DeployEnvVars from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits from stack_orchestrator.deploy.images import remote_tag_for_image_unique -DEFAULT_VOLUME_RESOURCES = Resources({ - "reservations": {"storage": "2Gi"} -}) +DEFAULT_VOLUME_RESOURCES = Resources({"reservations": {"storage": "2Gi"}}) -DEFAULT_CONTAINER_RESOURCES = Resources({ - "reservations": {"cpus": "0.1", "memory": "200M"}, - "limits": {"cpus": "1.0", "memory": "2000M"}, -}) +DEFAULT_CONTAINER_RESOURCES = Resources( + { + "reservations": {"cpus": "1.0", "memory": "2000M"}, + "limits": {"cpus": "4.0", "memory": "8000M"}, + } +) def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements: - def to_dict(limits: ResourceLimits): + def to_dict(limits: Optional[ResourceLimits]): if not limits: return None @@ -54,8 +65,7 @@ def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequi return ret return client.V1ResourceRequirements( - requests=to_dict(resources.reservations), - limits=to_dict(resources.limits) + requests=to_dict(resources.reservations), limits=to_dict(resources.limits) ) @@ -73,10 +83,14 @@ class ClusterInfo: self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) # Find the set of images in the pods self.image_set = images_for_deployment(pod_files) - self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) + # Filter out None values from env file + env_vars = { + k: v for k, v in env_var_map_from_file(compose_env_file).items() if v + } + self.environment_variables = DeployEnvVars(env_vars) self.app_name = deployment_name self.spec = spec - if (opts.o.debug): + if opts.o.debug: print(f"Env vars: {self.environment_variables.map}") def get_nodeports(self): @@ -90,31 +104,48 @@ class ClusterInfo: for raw_port in [str(p) for p in service_info["ports"]]: if opts.o.debug: print(f"service port: {raw_port}") - if ":" in raw_port: - parts = raw_port.split(":") + # Parse protocol suffix (e.g., "8001/udp" -> port=8001, + # protocol=UDP) + protocol = "TCP" + port_str = raw_port + if "/" in raw_port: + port_str, proto = raw_port.rsplit("/", 1) + protocol = proto.upper() + if ":" in port_str: + parts = port_str.split(":") if len(parts) != 2: raise Exception(f"Invalid port definition: {raw_port}") node_port = int(parts[0]) pod_port = int(parts[1]) else: node_port = None - pod_port = int(raw_port) + pod_port = int(port_str) service = client.V1Service( - metadata=client.V1ObjectMeta(name=f"{self.app_name}-nodeport-{pod_port}"), + metadata=client.V1ObjectMeta( + name=( + f"{self.app_name}-nodeport-" + f"{pod_port}-{protocol.lower()}" + ) + ), spec=client.V1ServiceSpec( type="NodePort", - ports=[client.V1ServicePort( - port=pod_port, - target_port=pod_port, - node_port=node_port - )], - selector={"app": self.app_name} - ) + ports=[ + client.V1ServicePort( + port=pod_port, + target_port=pod_port, + node_port=node_port, + protocol=protocol, + ) + ], + selector={"app": self.app_name}, + ), ) nodeports.append(service) return nodeports - def get_ingress(self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"): + def get_ingress( + self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod" + ): # No ingress for a deployment that has no http-proxy defined, for now http_proxy_info_list = self.spec.get_http_proxy() ingress = None @@ -126,10 +157,20 @@ class ClusterInfo: # TODO: good enough parsing for webapp deployment for now host_name = http_proxy_info["host-name"] rules = [] - tls = [client.V1IngressTLS( - hosts=certificate["spec"]["dnsNames"] if certificate else [host_name], - secret_name=certificate["spec"]["secretName"] if certificate else f"{self.app_name}-tls" - )] if use_tls else None + tls = ( + [ + client.V1IngressTLS( + hosts=certificate["spec"]["dnsNames"] + if certificate + else [host_name], + secret_name=certificate["spec"]["secretName"] + if certificate + else f"{self.app_name}-tls", + ) + ] + if use_tls + else None + ) paths = [] for route in http_proxy_info["routes"]: path = route["path"] @@ -138,65 +179,71 @@ class ClusterInfo: print(f"proxy config: {path} -> {proxy_to}") # proxy_to has the form : proxy_to_port = int(proxy_to.split(":")[1]) - paths.append(client.V1HTTPIngressPath( - path_type="Prefix", - path=path, - backend=client.V1IngressBackend( - service=client.V1IngressServiceBackend( - # TODO: this looks wrong - name=f"{self.app_name}-service", - # TODO: pull port number from the service - port=client.V1ServiceBackendPort(number=proxy_to_port) - ) + paths.append( + client.V1HTTPIngressPath( + path_type="Prefix", + path=path, + backend=client.V1IngressBackend( + service=client.V1IngressServiceBackend( + # TODO: this looks wrong + name=f"{self.app_name}-service", + # TODO: pull port number from the service + port=client.V1ServiceBackendPort(number=proxy_to_port), + ) + ), ) - )) - rules.append(client.V1IngressRule( - host=host_name, - http=client.V1HTTPIngressRuleValue( - paths=paths ) - )) - spec = client.V1IngressSpec( - tls=tls, - rules=rules + rules.append( + client.V1IngressRule( + host=host_name, http=client.V1HTTPIngressRuleValue(paths=paths) + ) ) + spec = client.V1IngressSpec(tls=tls, rules=rules) ingress_annotations = { - "kubernetes.io/ingress.class": "nginx", + "kubernetes.io/ingress.class": "caddy", } if not certificate: ingress_annotations["cert-manager.io/cluster-issuer"] = cluster_issuer ingress = client.V1Ingress( metadata=client.V1ObjectMeta( - name=f"{self.app_name}-ingress", - annotations=ingress_annotations + name=f"{self.app_name}-ingress", annotations=ingress_annotations ), - spec=spec + spec=spec, ) return ingress # TODO: suppoprt multiple services def get_service(self): - for pod_name in self.parsed_pod_yaml_map: - pod = self.parsed_pod_yaml_map[pod_name] - services = pod["services"] - for service_name in services: - service_info = services[service_name] - if "ports" in service_info: - port = int(service_info["ports"][0]) - if opts.o.debug: - print(f"service port: {port}") + # Collect all ports from http-proxy routes + ports_set = set() + http_proxy_list = self.spec.get_http_proxy() + if http_proxy_list: + for http_proxy in http_proxy_list: + for route in http_proxy.get("routes", []): + proxy_to = route.get("proxy-to", "") + if ":" in proxy_to: + port = int(proxy_to.split(":")[1]) + ports_set.add(port) + if opts.o.debug: + print(f"http-proxy route port: {port}") + + if not ports_set: + return None + + service_ports = [ + client.V1ServicePort(port=p, target_port=p, name=f"port-{p}") + for p in sorted(ports_set) + ] + service = client.V1Service( metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), spec=client.V1ServiceSpec( type="ClusterIP", - ports=[client.V1ServicePort( - port=port, - target_port=port - )], - selector={"app": self.app_name} - ) + ports=service_ports, + selector={"app": self.app_name}, + ), ) return service @@ -219,7 +266,7 @@ class ClusterInfo: labels = { "app": self.app_name, - "volume-label": f"{self.app_name}-{volume_name}" + "volume-label": f"{self.app_name}-{volume_name}", } if volume_path: storage_class_name = "manual" @@ -233,11 +280,13 @@ class ClusterInfo: access_modes=["ReadWriteOnce"], storage_class_name=storage_class_name, resources=to_k8s_resource_requirements(resources), - volume_name=k8s_volume_name + volume_name=k8s_volume_name, ) pvc = client.V1PersistentVolumeClaim( - metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", labels=labels), - spec=spec + metadata=client.V1ObjectMeta( + name=f"{self.app_name}-{volume_name}", labels=labels + ), + spec=spec, ) result.append(pvc) return result @@ -252,21 +301,28 @@ class ClusterInfo: print(f"{cfg_map_name} not in pod files") continue - if not cfg_map_path.startswith("/"): - cfg_map_path = os.path.join(os.path.dirname(self.spec.file_path), cfg_map_path) + if not cfg_map_path.startswith("/") and self.spec.file_path is not None: + cfg_map_path = os.path.join( + os.path.dirname(str(self.spec.file_path)), cfg_map_path + ) - # Read in all the files at a single-level of the directory. This mimics the behavior - # of `kubectl create configmap foo --from-file=/path/to/dir` + # Read in all the files at a single-level of the directory. + # This mimics the behavior of + # `kubectl create configmap foo --from-file=/path/to/dir` data = {} for f in os.listdir(cfg_map_path): full_path = os.path.join(cfg_map_path, f) if os.path.isfile(full_path): - data[f] = base64.b64encode(open(full_path, 'rb').read()).decode('ASCII') + data[f] = base64.b64encode(open(full_path, "rb").read()).decode( + "ASCII" + ) spec = client.V1ConfigMap( - metadata=client.V1ObjectMeta(name=f"{self.app_name}-{cfg_map_name}", - labels={"configmap-label": cfg_map_name}), - binary_data=data + metadata=client.V1ObjectMeta( + name=f"{self.app_name}-{cfg_map_name}", + labels={"configmap-label": cfg_map_name}, + ), + binary_data=data, ) result.append(spec) return result @@ -280,10 +336,14 @@ class ClusterInfo: resources = DEFAULT_VOLUME_RESOURCES for volume_name, volume_path in spec_volumes.items(): # We only need to create a volume if it is fully qualified HostPath. - # Otherwise, we create the PVC and expect the node to allocate the volume for us. + # Otherwise, we create the PVC and expect the node to allocate the volume + # for us. if not volume_path: if opts.o.debug: - print(f"{volume_name} does not require an explicit PersistentVolume, since it is not a bind-mount.") + print( + f"{volume_name} does not require an explicit " + "PersistentVolume, since it is not a bind-mount." + ) continue if volume_name not in named_volumes: @@ -292,30 +352,38 @@ class ClusterInfo: continue if not os.path.isabs(volume_path): - print(f"WARNING: {volume_name}:{volume_path} is not absolute, cannot bind volume.") + print( + f"WARNING: {volume_name}:{volume_path} is not absolute, " + "cannot bind volume." + ) continue if self.spec.is_kind_deployment(): - host_path = client.V1HostPathVolumeSource(path=get_kind_pv_bind_mount_path(volume_name)) + host_path = client.V1HostPathVolumeSource( + path=get_kind_pv_bind_mount_path(volume_name) + ) else: host_path = client.V1HostPathVolumeSource(path=volume_path) spec = client.V1PersistentVolumeSpec( storage_class_name="manual", access_modes=["ReadWriteOnce"], capacity=to_k8s_resource_requirements(resources).requests, - host_path=host_path + host_path=host_path, ) pv = client.V1PersistentVolume( - metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", - labels={"volume-label": f"{self.app_name}-{volume_name}"}), + metadata=client.V1ObjectMeta( + name=f"{self.app_name}-{volume_name}", + labels={"volume-label": f"{self.app_name}-{volume_name}"}, + ), spec=spec, ) result.append(pv) return result # TODO: put things like image pull policy into an object-scope struct - def get_deployment(self, image_pull_policy: str = None): + def get_deployment(self, image_pull_policy: Optional[str] = None): containers = [] + services = {} resources = self.spec.get_container_resources() if not resources: resources = DEFAULT_CONTAINER_RESOURCES @@ -326,42 +394,88 @@ class ClusterInfo: container_name = service_name service_info = services[service_name] image = service_info["image"] + container_ports = [] if "ports" in service_info: - port = int(service_info["ports"][0]) + for raw_port in [str(p) for p in service_info["ports"]]: + # Parse protocol suffix (e.g., "8001/udp" -> port=8001, + # protocol=UDP) + protocol = "TCP" + port_str = raw_port + if "/" in raw_port: + port_str, proto = raw_port.rsplit("/", 1) + protocol = proto.upper() + # Handle host:container port mapping - use container port + if ":" in port_str: + port_str = port_str.split(":")[-1] + port = int(port_str) + container_ports.append( + client.V1ContainerPort( + container_port=port, protocol=protocol + ) + ) if opts.o.debug: print(f"image: {image}") - print(f"service port: {port}") - merged_envs = merge_envs( - envs_from_compose_file( - service_info["environment"]), self.environment_variables.map - ) if "environment" in service_info else self.environment_variables.map + print(f"service ports: {container_ports}") + merged_envs = ( + merge_envs( + envs_from_compose_file( + service_info["environment"], self.environment_variables.map + ), + self.environment_variables.map, + ) + if "environment" in service_info + else self.environment_variables.map + ) envs = envs_from_environment_variables_map(merged_envs) if opts.o.debug: print(f"Merged envs: {envs}") # Re-write the image tag for remote deployment # Note self.app_name has the same value as deployment_id - image_to_use = remote_tag_for_image_unique( - image, - self.spec.get_image_registry(), - self.app_name) if self.spec.get_image_registry() is not None else image - volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name) + image_to_use = ( + remote_tag_for_image_unique( + image, self.spec.get_image_registry(), self.app_name + ) + if self.spec.get_image_registry() is not None + else image + ) + volume_mounts = volume_mounts_for_service( + self.parsed_pod_yaml_map, service_name + ) + # Handle command/entrypoint from compose file + # In docker-compose: entrypoint -> k8s command, command -> k8s args + container_command = None + container_args = None + if "entrypoint" in service_info: + entrypoint = service_info["entrypoint"] + container_command = ( + entrypoint if isinstance(entrypoint, list) else [entrypoint] + ) + if "command" in service_info: + cmd = service_info["command"] + container_args = cmd if isinstance(cmd, list) else cmd.split() container = client.V1Container( name=container_name, image=image_to_use, image_pull_policy=image_pull_policy, + command=container_command, + args=container_args, env=envs, - ports=[client.V1ContainerPort(container_port=port)], + ports=container_ports if container_ports else None, volume_mounts=volume_mounts, security_context=client.V1SecurityContext( privileged=self.spec.get_privileged(), capabilities=client.V1Capabilities( add=self.spec.get_capabilities() - ) if self.spec.get_capabilities() else None + ) + if self.spec.get_capabilities() + else None, ), resources=to_k8s_resource_requirements(resources), ) containers.append(container) - volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name) + volumes = volumes_for_pod_files( + self.parsed_pod_yaml_map, self.spec, self.app_name + ) image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")] annotations = None @@ -384,55 +498,55 @@ class ClusterInfo: affinities = [] for rule in self.spec.get_node_affinities(): # TODO add some input validation here - label_name = rule['label'] - label_value = rule['value'] - affinities.append(client.V1NodeSelectorTerm( - match_expressions=[client.V1NodeSelectorRequirement( - key=label_name, - operator="In", - values=[label_value] - )] - ) + label_name = rule["label"] + label_value = rule["value"] + affinities.append( + client.V1NodeSelectorTerm( + match_expressions=[ + client.V1NodeSelectorRequirement( + key=label_name, operator="In", values=[label_value] + ) + ] ) + ) affinity = client.V1Affinity( node_affinity=client.V1NodeAffinity( - required_during_scheduling_ignored_during_execution=client.V1NodeSelector( - node_selector_terms=affinities - )) + required_during_scheduling_ignored_during_execution=( + client.V1NodeSelector(node_selector_terms=affinities) + ) ) + ) if self.spec.get_node_tolerations(): tolerations = [] for toleration in self.spec.get_node_tolerations(): # TODO add some input validation here - toleration_key = toleration['key'] - toleration_value = toleration['value'] - tolerations.append(client.V1Toleration( - effect="NoSchedule", - key=toleration_key, - operator="Equal", - value=toleration_value - )) + toleration_key = toleration["key"] + toleration_value = toleration["value"] + tolerations.append( + client.V1Toleration( + effect="NoSchedule", + key=toleration_key, + operator="Equal", + value=toleration_value, + ) + ) template = client.V1PodTemplateSpec( - metadata=client.V1ObjectMeta( - annotations=annotations, - labels=labels - ), + metadata=client.V1ObjectMeta(annotations=annotations, labels=labels), spec=client.V1PodSpec( containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes, affinity=affinity, - tolerations=tolerations - ), + tolerations=tolerations, + runtime_class_name=self.spec.get_runtime_class(), + ), ) spec = client.V1DeploymentSpec( replicas=self.spec.get_replicas(), - template=template, selector={ - "matchLabels": - {"app": self.app_name} - } + template=template, + selector={"matchLabels": {"app": self.app_name}}, ) deployment = client.V1Deployment( diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index b254fd4c..3d0b697c 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -16,14 +16,29 @@ from datetime import datetime, timezone from pathlib import Path from kubernetes import client, config -from typing import List +from kubernetes.client.exceptions import ApiException +from typing import Any, Dict, List, Optional, cast from stack_orchestrator import constants from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator -from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind -from stack_orchestrator.deploy.k8s.helpers import install_ingress_for_kind, wait_for_ingress_in_kind -from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, containers_in_pod, log_stream_from_string -from stack_orchestrator.deploy.k8s.helpers import generate_kind_config +from stack_orchestrator.deploy.k8s.helpers import ( + create_cluster, + destroy_cluster, + load_images_into_kind, +) +from stack_orchestrator.deploy.k8s.helpers import ( + install_ingress_for_kind, + wait_for_ingress_in_kind, +) +from stack_orchestrator.deploy.k8s.helpers import ( + pods_in_deployment, + containers_in_pod, + log_stream_from_string, +) +from stack_orchestrator.deploy.k8s.helpers import ( + generate_kind_config, + generate_high_memlock_spec_json, +) from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo from stack_orchestrator.opts import opts from stack_orchestrator.deploy.deployment_context import DeploymentContext @@ -36,7 +51,7 @@ class AttrDict(dict): self.__dict__ = self -def _check_delete_exception(e: client.exceptions.ApiException): +def _check_delete_exception(e: ApiException) -> None: if e.status == 404: if opts.o.debug: print("Failed to delete object, continuing") @@ -44,6 +59,36 @@ def _check_delete_exception(e: client.exceptions.ApiException): error_exit(f"k8s api error: {e}") +def _create_runtime_class(name: str, handler: str): + """Create a RuntimeClass resource for custom containerd runtime handlers. + + RuntimeClass allows pods to specify which runtime handler to use, enabling + different pods to have different rlimit profiles (e.g., high-memlock). + + Args: + name: The name of the RuntimeClass resource + handler: The containerd runtime handler name + (must match containerdConfigPatches) + """ + api = client.NodeV1Api() + runtime_class = client.V1RuntimeClass( + api_version="node.k8s.io/v1", + kind="RuntimeClass", + metadata=client.V1ObjectMeta(name=name), + handler=handler, + ) + try: + api.create_runtime_class(runtime_class) + if opts.o.debug: + print(f"Created RuntimeClass: {name}") + except ApiException as e: + if e.status == 409: # Already exists + if opts.o.debug: + print(f"RuntimeClass {name} already exists") + else: + raise + + class K8sDeployer(Deployer): name: str = "k8s" type: str @@ -57,18 +102,31 @@ class K8sDeployer(Deployer): deployment_dir: Path deployment_context: DeploymentContext - def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None: + def __init__( + self, + type, + deployment_context: DeploymentContext, + compose_files, + compose_project_name, + compose_env_file, + ) -> None: self.type = type self.skip_cluster_management = False - # TODO: workaround pending refactoring above to cope with being created with a null deployment_context + # TODO: workaround pending refactoring above to cope with being + # created with a null deployment_context if deployment_context is None: return self.deployment_dir = deployment_context.deployment_dir self.deployment_context = deployment_context self.kind_cluster_name = compose_project_name self.cluster_info = ClusterInfo() - self.cluster_info.int(compose_files, compose_env_file, compose_project_name, deployment_context.spec) - if (opts.o.debug): + self.cluster_info.int( + compose_files, + compose_env_file, + compose_project_name, + deployment_context.spec, + ) + if opts.o.debug: print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Compose files: {compose_files}") print(f"Project name: {compose_project_name}") @@ -80,7 +138,11 @@ class K8sDeployer(Deployer): config.load_kube_config(context=f"kind-{self.kind_cluster_name}") else: # Get the config file and pass to load_kube_config() - config.load_kube_config(config_file=self.deployment_dir.joinpath(constants.kube_config_filename).as_posix()) + config.load_kube_config( + config_file=self.deployment_dir.joinpath( + constants.kube_config_filename + ).as_posix() + ) self.core_api = client.CoreV1Api() self.networking_api = client.NetworkingV1Api() self.apps_api = client.AppsV1Api() @@ -94,7 +156,9 @@ class K8sDeployer(Deployer): print(f"Sending this pv: {pv}") if not opts.o.dry_run: try: - pv_resp = self.core_api.read_persistent_volume(name=pv.metadata.name) + pv_resp = self.core_api.read_persistent_volume( + name=pv.metadata.name + ) if pv_resp: if opts.o.debug: print("PVs already present:") @@ -117,7 +181,8 @@ class K8sDeployer(Deployer): if not opts.o.dry_run: try: pvc_resp = self.core_api.read_namespaced_persistent_volume_claim( - name=pvc.metadata.name, namespace=self.k8s_namespace) + name=pvc.metadata.name, namespace=self.k8s_namespace + ) if pvc_resp: if opts.o.debug: print("PVCs already present:") @@ -126,7 +191,9 @@ class K8sDeployer(Deployer): except: # noqa: E722 pass - pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace) + pvc_resp = self.core_api.create_namespaced_persistent_volume_claim( + body=pvc, namespace=self.k8s_namespace + ) if opts.o.debug: print("PVCs created:") print(f"{pvc_resp}") @@ -138,8 +205,7 @@ class K8sDeployer(Deployer): print(f"Sending this ConfigMap: {cfg_map}") if not opts.o.dry_run: cfg_rsp = self.core_api.create_namespaced_config_map( - body=cfg_map, - namespace=self.k8s_namespace + body=cfg_map, namespace=self.k8s_namespace ) if opts.o.debug: print("ConfigMap created:") @@ -147,26 +213,37 @@ class K8sDeployer(Deployer): def _create_deployment(self): # Process compose files into a Deployment - deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always") + deployment = self.cluster_info.get_deployment( + image_pull_policy=None if self.is_kind() else "Always" + ) # Create the k8s objects if opts.o.debug: print(f"Sending this deployment: {deployment}") if not opts.o.dry_run: - deployment_resp = self.apps_api.create_namespaced_deployment( - body=deployment, namespace=self.k8s_namespace + deployment_resp = cast( + client.V1Deployment, + self.apps_api.create_namespaced_deployment( + body=deployment, namespace=self.k8s_namespace + ), ) if opts.o.debug: print("Deployment created:") - print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ - {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") + meta = deployment_resp.metadata + spec = deployment_resp.spec + if meta and spec and spec.template.spec: + ns = meta.namespace + name = meta.name + gen = meta.generation + containers = spec.template.spec.containers + img = containers[0].image if containers else None + print(f"{ns} {name} {gen} {img}") - service: client.V1Service = self.cluster_info.get_service() + service = self.cluster_info.get_service() if opts.o.debug: print(f"Sending this service: {service}") - if not opts.o.dry_run: + if service and not opts.o.dry_run: service_resp = self.core_api.create_namespaced_service( - namespace=self.k8s_namespace, - body=service + namespace=self.k8s_namespace, body=service ) if opts.o.debug: print("Service created:") @@ -177,7 +254,7 @@ class K8sDeployer(Deployer): group="cert-manager.io", version="v1", namespace=self.k8s_namespace, - plural="certificates" + plural="certificates", ) host_parts = host_name.split(".", 1) @@ -202,7 +279,9 @@ class K8sDeployer(Deployer): if before < now < after: # Check the status is Ready for condition in status.get("conditions", []): - if "True" == condition.get("status") and "Ready" == condition.get("type"): + if "True" == condition.get( + "status" + ) and "Ready" == condition.get("type"): return cert return None @@ -211,15 +290,27 @@ class K8sDeployer(Deployer): if not opts.o.dry_run: if self.is_kind() and not self.skip_cluster_management: # Create the kind cluster - create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) + create_cluster( + self.kind_cluster_name, + str(self.deployment_dir.joinpath(constants.kind_config_filename)), + ) # Ensure the referenced containers are copied into kind - load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) + load_images_into_kind( + self.kind_cluster_name, self.cluster_info.image_set + ) self.connect_api() if self.is_kind() and not self.skip_cluster_management: - # Now configure an ingress controller (not installed by default in kind) + # Configure ingress controller (not installed by default in kind) install_ingress_for_kind() - # Wait for ingress to start (deployment provisioning will fail unless this is done) + # Wait for ingress to start + # (deployment provisioning will fail unless this is done) wait_for_ingress_in_kind() + # Create RuntimeClass if unlimited_memlock is enabled + if self.cluster_info.spec.get_unlimited_memlock(): + _create_runtime_class( + constants.high_memlock_runtime, + constants.high_memlock_runtime, + ) else: print("Dry run mode enabled, skipping k8s API connect") @@ -228,21 +319,26 @@ class K8sDeployer(Deployer): self._create_deployment() http_proxy_info = self.cluster_info.spec.get_http_proxy() - # Note: at present we don't support tls for kind (and enabling tls causes errors) + # Note: we don't support tls for kind (enabling tls causes errors) use_tls = http_proxy_info and not self.is_kind() - certificate = self._find_certificate_for_host_name(http_proxy_info[0]["host-name"]) if use_tls else None + certificate = ( + self._find_certificate_for_host_name(http_proxy_info[0]["host-name"]) + if use_tls + else None + ) if opts.o.debug: if certificate: print(f"Using existing certificate: {certificate}") - ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=use_tls, certificate=certificate) + ingress = self.cluster_info.get_ingress( + use_tls=use_tls, certificate=certificate + ) if ingress: if opts.o.debug: print(f"Sending this ingress: {ingress}") if not opts.o.dry_run: ingress_resp = self.networking_api.create_namespaced_ingress( - namespace=self.k8s_namespace, - body=ingress + namespace=self.k8s_namespace, body=ingress ) if opts.o.debug: print("Ingress created:") @@ -257,8 +353,7 @@ class K8sDeployer(Deployer): print(f"Sending this nodeport: {nodeport}") if not opts.o.dry_run: nodeport_resp = self.core_api.create_namespaced_service( - namespace=self.k8s_namespace, - body=nodeport + namespace=self.k8s_namespace, body=nodeport ) if opts.o.debug: print("NodePort created:") @@ -276,11 +371,13 @@ class K8sDeployer(Deployer): if opts.o.debug: print(f"Deleting this pv: {pv}") try: - pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name) + pv_resp = self.core_api.delete_persistent_volume( + name=pv.metadata.name + ) if opts.o.debug: print("PV deleted:") print(f"{pv_resp}") - except client.exceptions.ApiException as e: + except ApiException as e: _check_delete_exception(e) # Figure out the PVCs for this deployment @@ -295,7 +392,7 @@ class K8sDeployer(Deployer): if opts.o.debug: print("PVCs deleted:") print(f"{pvc_resp}") - except client.exceptions.ApiException as e: + except ApiException as e: _check_delete_exception(e) # Figure out the ConfigMaps for this deployment @@ -310,39 +407,40 @@ class K8sDeployer(Deployer): if opts.o.debug: print("ConfigMap deleted:") print(f"{cfg_map_resp}") - except client.exceptions.ApiException as e: + except ApiException as e: _check_delete_exception(e) deployment = self.cluster_info.get_deployment() if opts.o.debug: print(f"Deleting this deployment: {deployment}") - try: - self.apps_api.delete_namespaced_deployment( - name=deployment.metadata.name, namespace=self.k8s_namespace - ) - except client.exceptions.ApiException as e: - _check_delete_exception(e) + if deployment and deployment.metadata and deployment.metadata.name: + try: + self.apps_api.delete_namespaced_deployment( + name=deployment.metadata.name, namespace=self.k8s_namespace + ) + except ApiException as e: + _check_delete_exception(e) - service: client.V1Service = self.cluster_info.get_service() + service = self.cluster_info.get_service() if opts.o.debug: print(f"Deleting service: {service}") - try: - self.core_api.delete_namespaced_service( - namespace=self.k8s_namespace, - name=service.metadata.name - ) - except client.exceptions.ApiException as e: - _check_delete_exception(e) + if service and service.metadata and service.metadata.name: + try: + self.core_api.delete_namespaced_service( + namespace=self.k8s_namespace, name=service.metadata.name + ) + except ApiException as e: + _check_delete_exception(e) - ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind()) - if ingress: + ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind()) + if ingress and ingress.metadata and ingress.metadata.name: if opts.o.debug: print(f"Deleting this ingress: {ingress}") try: self.networking_api.delete_namespaced_ingress( name=ingress.metadata.name, namespace=self.k8s_namespace ) - except client.exceptions.ApiException as e: + except ApiException as e: _check_delete_exception(e) else: if opts.o.debug: @@ -352,13 +450,13 @@ class K8sDeployer(Deployer): for nodeport in nodeports: if opts.o.debug: print(f"Deleting this nodeport: {nodeport}") - try: - self.core_api.delete_namespaced_service( - namespace=self.k8s_namespace, - name=nodeport.metadata.name - ) - except client.exceptions.ApiException as e: - _check_delete_exception(e) + if nodeport.metadata and nodeport.metadata.name: + try: + self.core_api.delete_namespaced_service( + namespace=self.k8s_namespace, name=nodeport.metadata.name + ) + except ApiException as e: + _check_delete_exception(e) else: if opts.o.debug: print("No nodeport to delete") @@ -375,8 +473,9 @@ class K8sDeployer(Deployer): if all_pods.items: for p in all_pods.items: - if f"{self.cluster_info.app_name}-deployment" in p.metadata.name: - pods.append(p) + if p.metadata and p.metadata.name: + if f"{self.cluster_info.app_name}-deployment" in p.metadata.name: + pods.append(p) if not pods: return @@ -385,21 +484,40 @@ class K8sDeployer(Deployer): ip = "?" tls = "?" try: - ingress = self.networking_api.read_namespaced_ingress(namespace=self.k8s_namespace, - name=self.cluster_info.get_ingress().metadata.name) + cluster_ingress = self.cluster_info.get_ingress() + if cluster_ingress is None or cluster_ingress.metadata is None: + return + ingress = cast( + client.V1Ingress, + self.networking_api.read_namespaced_ingress( + namespace=self.k8s_namespace, + name=cluster_ingress.metadata.name, + ), + ) + if not ingress.spec or not ingress.spec.tls or not ingress.spec.rules: + return - cert = self.custom_obj_api.get_namespaced_custom_object( - group="cert-manager.io", - version="v1", - namespace=self.k8s_namespace, - plural="certificates", - name=ingress.spec.tls[0].secret_name + cert = cast( + Dict[str, Any], + self.custom_obj_api.get_namespaced_custom_object( + group="cert-manager.io", + version="v1", + namespace=self.k8s_namespace, + plural="certificates", + name=ingress.spec.tls[0].secret_name, + ), ) hostname = ingress.spec.rules[0].host - ip = ingress.status.load_balancer.ingress[0].ip + if ingress.status and ingress.status.load_balancer: + lb_ingress = ingress.status.load_balancer.ingress + if lb_ingress: + ip = lb_ingress[0].ip or "?" + cert_status = cert.get("status", {}) tls = "notBefore: %s; notAfter: %s; names: %s" % ( - cert["status"]["notBefore"], cert["status"]["notAfter"], ingress.spec.tls[0].hosts + cert_status.get("notBefore", "?"), + cert_status.get("notAfter", "?"), + ingress.spec.tls[0].hosts, ) except: # noqa: E722 pass @@ -412,10 +530,16 @@ class K8sDeployer(Deployer): print("Pods:") for p in pods: + if not p.metadata: + continue + ns = p.metadata.namespace + name = p.metadata.name if p.metadata.deletion_timestamp: - print(f"\t{p.metadata.namespace}/{p.metadata.name}: Terminating ({p.metadata.deletion_timestamp})") + ts = p.metadata.deletion_timestamp + print(f"\t{ns}/{name}: Terminating ({ts})") else: - print(f"\t{p.metadata.namespace}/{p.metadata.name}: Running ({p.metadata.creation_timestamp})") + ts = p.metadata.creation_timestamp + print(f"\t{ns}/{name}: Running ({ts})") def ps(self): self.connect_api() @@ -430,19 +554,22 @@ class K8sDeployer(Deployer): for c in p.spec.containers: if c.ports: for prt in c.ports: - ports[str(prt.container_port)] = [AttrDict({ - "HostIp": pod_ip, - "HostPort": prt.container_port - })] + ports[str(prt.container_port)] = [ + AttrDict( + {"HostIp": pod_ip, "HostPort": prt.container_port} + ) + ] - ret.append(AttrDict({ - "id": f"{p.metadata.namespace}/{p.metadata.name}", - "name": p.metadata.name, - "namespace": p.metadata.namespace, - "network_settings": AttrDict({ - "ports": ports - }) - })) + ret.append( + AttrDict( + { + "id": f"{p.metadata.namespace}/{p.metadata.name}", + "name": p.metadata.name, + "namespace": p.metadata.namespace, + "network_settings": AttrDict({"ports": ports}), + } + ) + ) return ret @@ -465,15 +592,17 @@ class K8sDeployer(Deployer): else: k8s_pod_name = pods[0] containers = containers_in_pod(self.core_api, k8s_pod_name) - # If the pod is not yet started, the logs request below will throw an exception + # If pod not started, logs request below will throw an exception try: log_data = "" for container in containers: - container_log = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container=container) + container_log = self.core_api.read_namespaced_pod_log( + k8s_pod_name, namespace="default", container=container + ) container_log_lines = container_log.splitlines() for line in container_log_lines: log_data += f"{container}: {line}\n" - except client.exceptions.ApiException as e: + except ApiException as e: if opts.o.debug: print(f"Error from read_namespaced_pod_log: {e}") log_data = "******* No logs available ********\n" @@ -482,34 +611,85 @@ class K8sDeployer(Deployer): def update(self): self.connect_api() ref_deployment = self.cluster_info.get_deployment() + if not ref_deployment or not ref_deployment.metadata: + return + ref_name = ref_deployment.metadata.name + if not ref_name: + return - deployment = self.apps_api.read_namespaced_deployment( - name=ref_deployment.metadata.name, - namespace=self.k8s_namespace + deployment = cast( + client.V1Deployment, + self.apps_api.read_namespaced_deployment( + name=ref_name, namespace=self.k8s_namespace + ), ) + if not deployment.spec or not deployment.spec.template: + return + template_spec = deployment.spec.template.spec + if not template_spec or not template_spec.containers: + return - new_env = ref_deployment.spec.template.spec.containers[0].env - for container in deployment.spec.template.spec.containers: - old_env = container.env - if old_env != new_env: - container.env = new_env + ref_spec = ref_deployment.spec + if ref_spec and ref_spec.template and ref_spec.template.spec: + ref_containers = ref_spec.template.spec.containers + if ref_containers: + new_env = ref_containers[0].env + for container in template_spec.containers: + old_env = container.env + if old_env != new_env: + container.env = new_env - deployment.spec.template.metadata.annotations = { - "kubectl.kubernetes.io/restartedAt": datetime.utcnow() - .replace(tzinfo=timezone.utc) - .isoformat() - } + template_meta = deployment.spec.template.metadata + if template_meta: + template_meta.annotations = { + "kubectl.kubernetes.io/restartedAt": datetime.utcnow() + .replace(tzinfo=timezone.utc) + .isoformat() + } self.apps_api.patch_namespaced_deployment( - name=ref_deployment.metadata.name, + name=ref_name, namespace=self.k8s_namespace, - body=deployment + body=deployment, ) - def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): + def run( + self, + image: str, + command=None, + user=None, + volumes=None, + entrypoint=None, + env={}, + ports=[], + detach=False, + ): # We need to figure out how to do this -- check why we're being called first pass + def run_job(self, job_name: str, helm_release: Optional[str] = None): + if not opts.o.dry_run: + from stack_orchestrator.deploy.k8s.helm.job_runner import run_helm_job + + # Check if this is a helm-based deployment + chart_dir = self.deployment_dir / "chart" + if not chart_dir.exists(): + # TODO: Implement job support for compose-based K8s deployments + raise Exception( + f"Job support is only available for helm-based " + f"deployments. Chart directory not found: {chart_dir}" + ) + + # Run the job using the helm job runner + run_helm_job( + chart_dir=chart_dir, + job_name=job_name, + release=helm_release, + namespace=self.k8s_namespace, + timeout=600, + verbose=opts.o.verbose, + ) + def is_kind(self): return self.type == "k8s-kind" @@ -525,6 +705,20 @@ class K8sDeployerConfigGenerator(DeployerConfigGenerator): def generate(self, deployment_dir: Path): # No need to do this for the remote k8s case if self.type == "k8s-kind": + # Generate high-memlock-spec.json if unlimited_memlock is enabled. + # Must be done before generate_kind_config() which references it. + if self.deployment_context.spec.get_unlimited_memlock(): + spec_content = generate_high_memlock_spec_json() + spec_file = deployment_dir.joinpath( + constants.high_memlock_spec_filename + ) + if opts.o.debug: + print( + f"Creating high-memlock spec for unlimited memlock: {spec_file}" + ) + with open(spec_file, "w") as output_file: + output_file.write(spec_content) + # Check the file isn't already there # Get the config file contents content = generate_kind_config(deployment_dir, self.deployment_context) diff --git a/stack_orchestrator/deploy/k8s/helm/__init__.py b/stack_orchestrator/deploy/k8s/helm/__init__.py new file mode 100644 index 00000000..3d935105 --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/__init__.py @@ -0,0 +1,14 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . diff --git a/stack_orchestrator/deploy/k8s/helm/chart_generator.py b/stack_orchestrator/deploy/k8s/helm/chart_generator.py new file mode 100644 index 00000000..7e9c974e --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/chart_generator.py @@ -0,0 +1,336 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from pathlib import Path + +from stack_orchestrator import constants +from stack_orchestrator.opts import opts +from stack_orchestrator.util import ( + get_parsed_stack_config, + get_pod_list, + get_pod_file_path, + get_job_list, + get_job_file_path, + error_exit, +) +from stack_orchestrator.deploy.k8s.helm.kompose_wrapper import ( + check_kompose_available, + get_kompose_version, + convert_to_helm_chart, +) +from stack_orchestrator.util import get_yaml + + +def _wrap_job_templates_with_conditionals(chart_dir: Path, jobs: list) -> None: + """ + Wrap job templates with conditional checks so they are not created by default. + Jobs will only be created when explicitly enabled via --set jobs..enabled=true + """ + templates_dir = chart_dir / "templates" + if not templates_dir.exists(): + return + + for job_name in jobs: + # Find job template file (kompose generates -job.yaml) + job_template_file = templates_dir / f"{job_name}-job.yaml" + + if not job_template_file.exists(): + if opts.o.debug: + print(f"Warning: Job template not found: {job_template_file}") + continue + + # Read the template content + content = job_template_file.read_text() + + # Wrap with conditional (default false) + # Use 'index' function to handle job names with dashes + # Provide default dict for .Values.jobs to handle case where it doesn't exist + condition = ( + f"{{{{- if (index (.Values.jobs | default dict) " + f'"{job_name}" | default dict).enabled | default false }}}}' + ) + wrapped_content = f"""{condition} +{content}{{{{- end }}}} +""" + + # Write back + job_template_file.write_text(wrapped_content) + + if opts.o.debug: + print(f"Wrapped job template with conditional: {job_template_file.name}") + + +def _post_process_chart(chart_dir: Path, chart_name: str, jobs: list) -> None: + """ + Post-process Kompose-generated chart to fix common issues. + + Fixes: + 1. Chart.yaml name, description and keywords + 2. Add conditional wrappers to job templates (default: disabled) + + TODO: + - Add defaultMode: 0755 to ConfigMap volumes containing scripts (.sh files) + """ + yaml = get_yaml() + + # Fix Chart.yaml + chart_yaml_path = chart_dir / "Chart.yaml" + if chart_yaml_path.exists(): + chart_yaml = yaml.load(open(chart_yaml_path, "r")) + + # Fix name + chart_yaml["name"] = chart_name + + # Fix description + chart_yaml["description"] = f"Generated Helm chart for {chart_name} stack" + + # Fix keywords + if "keywords" in chart_yaml and isinstance(chart_yaml["keywords"], list): + chart_yaml["keywords"] = [chart_name] + + with open(chart_yaml_path, "w") as f: + yaml.dump(chart_yaml, f) + + # Process job templates: wrap with conditionals (default disabled) + if jobs: + _wrap_job_templates_with_conditionals(chart_dir, jobs) + + +def generate_helm_chart( + stack_path: str, spec_file: str, deployment_dir_path: Path +) -> None: + """ + Generate a self-sufficient Helm chart from stack compose files using Kompose. + + Args: + stack_path: Path to the stack directory + spec_file: Path to the deployment spec file + deployment_dir_path: Deployment directory path + (already created with deployment.yml) + + Output structure: + deployment-dir/ + ├── deployment.yml # Contains cluster-id + ├── spec.yml # Reference + ├── stack.yml # Reference + └── chart/ # Self-sufficient Helm chart + ├── Chart.yaml + ├── README.md + └── templates/ + └── *.yaml + + TODO: Enhancements: + - Convert Deployments to StatefulSets for stateful services (zenithd, postgres) + - Add _helpers.tpl with common label/selector functions + - Enhance Chart.yaml with proper metadata (version, description, etc.) + """ + + parsed_stack = get_parsed_stack_config(stack_path) + if parsed_stack is None: + error_exit(f"Failed to parse stack config: {stack_path}") + stack_name = parsed_stack.get("name", stack_path) + + # 1. Check Kompose availability + if not check_kompose_available(): + error_exit("kompose not found in PATH.\n") + + # 2. Read cluster-id from deployment.yml + deployment_file = deployment_dir_path / constants.deployment_file_name + if not deployment_file.exists(): + error_exit(f"Deployment file not found: {deployment_file}") + + yaml = get_yaml() + deployment_config = yaml.load(open(deployment_file, "r")) + cluster_id = deployment_config.get(constants.cluster_id_key) + if not cluster_id: + error_exit(f"cluster-id not found in {deployment_file}") + + # 3. Derive chart name from stack name + cluster-id suffix + # Sanitize stack name for use in chart name + sanitized_stack_name = stack_name.replace("_", "-").replace(" ", "-") + + # Extract hex suffix from cluster-id (after the prefix) + # cluster-id format: "laconic-" -> extract the hex part + cluster_id_suffix = cluster_id.split("-", 1)[1] if "-" in cluster_id else cluster_id + + # Combine to create human-readable + unique chart name + chart_name = f"{sanitized_stack_name}-{cluster_id_suffix}" + + if opts.o.debug: + print(f"Cluster ID: {cluster_id}") + print(f"Chart name: {chart_name}") + + # 4. Get compose files from stack (pods + jobs) + pods = get_pod_list(parsed_stack) + if not pods: + error_exit(f"No pods found in stack: {stack_path}") + + jobs = get_job_list(parsed_stack) + + if opts.o.debug: + print(f"Found {len(pods)} pod(s) in stack: {pods}") + if jobs: + print(f"Found {len(jobs)} job(s) in stack: {jobs}") + + compose_files = [] + for pod in pods: + pod_file = get_pod_file_path(stack_path, parsed_stack, pod) + if pod_file is None: + error_exit(f"Pod file path not found for pod: {pod}") + pod_file_path = Path(pod_file) if isinstance(pod_file, str) else pod_file + if not pod_file_path.exists(): + error_exit(f"Pod file not found: {pod_file_path}") + compose_files.append(pod_file_path) + if opts.o.debug: + print(f"Found compose file: {pod_file_path.name}") + + # Add job compose files + job_files = [] + for job in jobs: + job_file = get_job_file_path(stack_path, parsed_stack, job) + if job_file is None: + error_exit(f"Job file path not found for job: {job}") + job_file_path = Path(job_file) if isinstance(job_file, str) else job_file + if not job_file_path.exists(): + error_exit(f"Job file not found: {job_file_path}") + compose_files.append(job_file_path) + job_files.append(job_file_path) + if opts.o.debug: + print(f"Found job compose file: {job_file_path.name}") + + try: + version = get_kompose_version() + print(f"Using kompose version: {version}") + except Exception as e: + error_exit(f"Failed to get kompose version: {e}") + + # 5. Create chart directory and invoke Kompose + chart_dir = deployment_dir_path / "chart" + + print( + f"Converting {len(compose_files)} compose file(s) to Helm chart " + "using Kompose..." + ) + + try: + output = convert_to_helm_chart( + compose_files=compose_files, output_dir=chart_dir, chart_name=chart_name + ) + if opts.o.debug: + print(f"Kompose output:\n{output}") + except Exception as e: + error_exit(f"Helm chart generation failed: {e}") + + # 6. Post-process generated chart + _post_process_chart(chart_dir, chart_name, jobs) + + # 7. Generate README.md with basic installation instructions + readme_content = f"""# {chart_name} Helm Chart + +Generated by laconic-so from stack: `{stack_path}` + +## Prerequisites + +- Kubernetes cluster (v1.27+) +- Helm (v3.12+) +- kubectl configured to access your cluster + +## Installation + +```bash +# Install the chart +helm install {chart_name} {chart_dir} + +# Alternatively, install with your own release name +# helm install {chart_dir} + +# Check deployment status +kubectl get pods +``` + +## Upgrade + +To apply changes made to chart, perform upgrade: + +```bash +helm upgrade {chart_name} {chart_dir} +``` + +## Uninstallation + +```bash +helm uninstall {chart_name} +``` + +## Configuration + +The chart was generated from Docker Compose files using Kompose. + +### Customization + +Edit the generated template files in `templates/` to customize: +- Image repositories and tags +- Resource limits (CPU, memory) +- Persistent volume sizes +- Replica counts +""" + + readme_path = chart_dir / "README.md" + readme_path.write_text(readme_content) + + if opts.o.debug: + print(f"Generated README: {readme_path}") + + # 7. Success message + print(f"\n{'=' * 60}") + print("✓ Helm chart generated successfully!") + print(f"{'=' * 60}") + print("\nChart details:") + print(f" Name: {chart_name}") + print(f" Location: {chart_dir.absolute()}") + print(f" Stack: {stack_path}") + + # Count generated files + template_files = ( + list((chart_dir / "templates").glob("*.yaml")) + if (chart_dir / "templates").exists() + else [] + ) + print(f" Files: {len(template_files)} template(s) generated") + + print("\nDeployment directory structure:") + print(f" {deployment_dir_path}/") + print(" ├── deployment.yml (cluster-id)") + print(" ├── spec.yml (reference)") + print(" ├── stack.yml (reference)") + print(" └── chart/ (self-sufficient Helm chart)") + + print("\nNext steps:") + print(" 1. Review the chart:") + print(f" cd {chart_dir}") + print(" cat Chart.yaml") + print("") + print(" 2. Review generated templates:") + print(" ls templates/") + print("") + print(" 3. Install to Kubernetes:") + print(f" helm install {chart_name} {chart_dir}") + print("") + print(" # Or use your own release name") + print(f" helm install {chart_dir}") + print("") + print(" 4. Check deployment:") + print(" kubectl get pods") + print("") diff --git a/stack_orchestrator/deploy/k8s/helm/job_runner.py b/stack_orchestrator/deploy/k8s/helm/job_runner.py new file mode 100644 index 00000000..9f34ce6c --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/job_runner.py @@ -0,0 +1,171 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import subprocess +import tempfile +import os +import json +from pathlib import Path +from typing import Optional +from stack_orchestrator.util import get_yaml + + +def get_release_name_from_chart(chart_dir: Path) -> str: + """ + Read the chart name from Chart.yaml to use as the release name. + + Args: + chart_dir: Path to the Helm chart directory + + Returns: + Chart name from Chart.yaml + + Raises: + Exception if Chart.yaml not found or name is missing + """ + chart_yaml_path = chart_dir / "Chart.yaml" + if not chart_yaml_path.exists(): + raise Exception(f"Chart.yaml not found: {chart_yaml_path}") + + yaml = get_yaml() + chart_yaml = yaml.load(open(chart_yaml_path, "r")) + + if "name" not in chart_yaml: + raise Exception(f"Chart name not found in {chart_yaml_path}") + + return chart_yaml["name"] + + +def run_helm_job( + chart_dir: Path, + job_name: str, + release: Optional[str] = None, + namespace: str = "default", + timeout: int = 600, + verbose: bool = False, +) -> None: + """ + Run a one-time job from a Helm chart. + + This function: + 1. Uses provided release name, or reads it from Chart.yaml if not provided + 2. Uses helm template to render the job manifest with the job enabled + 3. Applies the job manifest to the cluster + 4. Waits for the job to complete + + Args: + chart_dir: Path to the Helm chart directory + job_name: Name of the job to run (without -job suffix) + release: Optional Helm release name (defaults to chart name from Chart.yaml) + namespace: Kubernetes namespace + timeout: Timeout in seconds for job completion (default: 600) + verbose: Enable verbose output + + Raises: + Exception if the job fails or times out + """ + if not chart_dir.exists(): + raise Exception(f"Chart directory not found: {chart_dir}") + + # Use provided release name, or get it from Chart.yaml + if release is None: + release = get_release_name_from_chart(chart_dir) + if verbose: + print(f"Using release name from Chart.yaml: {release}") + else: + if verbose: + print(f"Using provided release name: {release}") + + job_template_file = f"templates/{job_name}-job.yaml" + + if verbose: + print(f"Running job '{job_name}' from helm chart: {chart_dir}") + + # Use helm template to render the job manifest + with tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False + ) as tmp_file: + try: + # Render job template with job enabled + # Use --set-json to properly handle job names with dashes + jobs_dict = {job_name: {"enabled": True}} + values_json = json.dumps(jobs_dict) + helm_cmd = [ + "helm", + "template", + release, + str(chart_dir), + "--show-only", + job_template_file, + "--set-json", + f"jobs={values_json}", + ] + + if verbose: + print(f"Running: {' '.join(helm_cmd)}") + + result = subprocess.run( + helm_cmd, check=True, capture_output=True, text=True + ) + tmp_file.write(result.stdout) + tmp_file.flush() + + if verbose: + print(f"Generated job manifest:\n{result.stdout}") + + # Parse the manifest to get the actual job name + yaml = get_yaml() + manifest = yaml.load(result.stdout) + actual_job_name = manifest.get("metadata", {}).get("name", job_name) + + # Apply the job manifest + kubectl_apply_cmd = [ + "kubectl", + "apply", + "-f", + tmp_file.name, + "-n", + namespace, + ] + subprocess.run( + kubectl_apply_cmd, check=True, capture_output=True, text=True + ) + + if verbose: + print(f"Job {actual_job_name} created, waiting for completion...") + + # Wait for job completion + wait_cmd = [ + "kubectl", + "wait", + "--for=condition=complete", + f"job/{actual_job_name}", + f"--timeout={timeout}s", + "-n", + namespace, + ] + + subprocess.run(wait_cmd, check=True, capture_output=True, text=True) + + if verbose: + print(f"Job {job_name} completed successfully") + + except subprocess.CalledProcessError as e: + error_msg = e.stderr if e.stderr else str(e) + raise Exception(f"Job failed: {error_msg}") + finally: + # Clean up temp file + if os.path.exists(tmp_file.name): + os.unlink(tmp_file.name) diff --git a/stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py b/stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py new file mode 100644 index 00000000..520a668e --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py @@ -0,0 +1,103 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import subprocess +import shutil +from pathlib import Path +from typing import List, Optional + + +def check_kompose_available() -> bool: + """Check if kompose binary is available in PATH.""" + return shutil.which("kompose") is not None + + +def get_kompose_version() -> str: + """ + Get the installed kompose version. + + Returns: + Version string (e.g., "1.34.0") + + Raises: + Exception if kompose is not available + """ + if not check_kompose_available(): + raise Exception("kompose not found in PATH") + + result = subprocess.run( + ["kompose", "version"], capture_output=True, text=True, timeout=10 + ) + + if result.returncode != 0: + raise Exception(f"Failed to get kompose version: {result.stderr}") + + # Parse version from output like "1.34.0 (HEAD)" + # Output format: "1.34.0 (HEAD)" or just "1.34.0" + version_line = result.stdout.strip() + version = version_line.split()[0] if version_line else "unknown" + + return version + + +def convert_to_helm_chart( + compose_files: List[Path], output_dir: Path, chart_name: Optional[str] = None +) -> str: + """ + Invoke kompose to convert Docker Compose files to a Helm chart. + + Args: + compose_files: List of paths to docker-compose.yml files + output_dir: Directory where the Helm chart will be generated + chart_name: Optional name for the chart (defaults to directory name) + + Returns: + stdout from kompose command + + Raises: + Exception if kompose conversion fails + """ + if not check_kompose_available(): + raise Exception( + "kompose not found in PATH. " + "Install from: https://kompose.io/installation/" + ) + + # Ensure output directory exists + output_dir.mkdir(parents=True, exist_ok=True) + + # Build kompose command + cmd = ["kompose", "convert"] + + # Add all compose files + for compose_file in compose_files: + if not compose_file.exists(): + raise Exception(f"Compose file not found: {compose_file}") + cmd.extend(["-f", str(compose_file)]) + + # Add chart flag and output directory + cmd.extend(["--chart", "-o", str(output_dir)]) + + # Execute kompose + result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) + + if result.returncode != 0: + raise Exception( + f"Kompose conversion failed:\n" + f"Command: {' '.join(cmd)}\n" + f"Error: {result.stderr}" + ) + + return result.stdout diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 80fb9c6a..a125d4f5 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -18,12 +18,31 @@ import os from pathlib import Path import subprocess import re -from typing import Set, Mapping, List +from typing import Set, Mapping, List, Optional, cast from stack_orchestrator.util import get_k8s_dir, error_exit from stack_orchestrator.opts import opts from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names from stack_orchestrator.deploy.deployer import DeployerException +from stack_orchestrator import constants + + +def get_kind_cluster(): + """Get an existing kind cluster, if any. + + Uses `kind get clusters` to find existing clusters. + Returns the cluster name or None if no cluster exists. + """ + result = subprocess.run( + "kind get clusters", shell=True, capture_output=True, text=True + ) + if result.returncode != 0: + return None + + clusters = result.stdout.strip().splitlines() + if clusters: + return clusters[0] # Return the first cluster found + return None def _run_command(command: str): @@ -50,38 +69,53 @@ def wait_for_ingress_in_kind(): for i in range(20): warned_waiting = False w = watch.Watch() - for event in w.stream(func=core_v1.list_namespaced_pod, - namespace="ingress-nginx", - label_selector="app.kubernetes.io/component=controller", - timeout_seconds=30): - if event['object'].status.container_statuses: - if event['object'].status.container_statuses[0].ready is True: + for event in w.stream( + func=core_v1.list_namespaced_pod, + namespace="caddy-system", + label_selector=( + "app.kubernetes.io/name=caddy-ingress-controller," + "app.kubernetes.io/component=controller" + ), + timeout_seconds=30, + ): + event_dict = cast(dict, event) + pod = cast(client.V1Pod, event_dict.get("object")) + if pod and pod.status and pod.status.container_statuses: + if pod.status.container_statuses[0].ready is True: if warned_waiting: - print("Ingress controller is ready") + print("Caddy ingress controller is ready") return - print("Waiting for ingress controller to become ready...") + print("Waiting for Caddy ingress controller to become ready...") warned_waiting = True - error_exit("ERROR: Timed out waiting for ingress to become ready") + error_exit("ERROR: Timed out waiting for Caddy ingress to become ready") def install_ingress_for_kind(): api_client = client.ApiClient() - ingress_install = os.path.abspath(get_k8s_dir().joinpath("components", "ingress", "ingress-nginx-kind-deploy.yaml")) + ingress_install = os.path.abspath( + get_k8s_dir().joinpath( + "components", "ingress", "ingress-caddy-kind-deploy.yaml" + ) + ) if opts.o.debug: - print("Installing nginx ingress controller in kind cluster") + print("Installing Caddy ingress controller in kind cluster") utils.create_from_yaml(api_client, yaml_file=ingress_install) def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]): for image in image_set: - result = _run_command(f"kind load docker-image {image} --name {kind_cluster_name}") + result = _run_command( + f"kind load docker-image {image} --name {kind_cluster_name}" + ) if result.returncode != 0: - raise DeployerException(f"kind create cluster failed: {result}") + raise DeployerException(f"kind load docker-image failed: {result}") def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str): pods = [] - pod_response = core_api.list_namespaced_pod(namespace="default", label_selector=f"app={deployment_name}") + pod_response = core_api.list_namespaced_pod( + namespace="default", label_selector=f"app={deployment_name}" + ) if opts.o.debug: print(f"pod_response: {pod_response}") for pod_info in pod_response.items: @@ -90,14 +124,18 @@ def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str): return pods -def containers_in_pod(core_api: client.CoreV1Api, pod_name: str): - containers = [] - pod_response = core_api.read_namespaced_pod(pod_name, namespace="default") +def containers_in_pod(core_api: client.CoreV1Api, pod_name: str) -> List[str]: + containers: List[str] = [] + pod_response = cast( + client.V1Pod, core_api.read_namespaced_pod(pod_name, namespace="default") + ) if opts.o.debug: print(f"pod_response: {pod_response}") - pod_containers = pod_response.spec.containers - for pod_container in pod_containers: - containers.append(pod_container.name) + if not pod_response.spec or not pod_response.spec.containers: + return containers + for pod_container in pod_response.spec.containers: + if pod_container.name: + containers.append(pod_container.name) return containers @@ -137,13 +175,16 @@ def volume_mounts_for_service(parsed_pod_files, service): if "volumes" in service_obj: volumes = service_obj["volumes"] for mount_string in volumes: - # Looks like: test-data:/data or test-data:/data:ro or test-data:/data:rw + # Looks like: test-data:/data + # or test-data:/data:ro or test-data:/data:rw if opts.o.debug: print(f"mount_string: {mount_string}") mount_split = mount_string.split(":") volume_name = mount_split[0] mount_path = mount_split[1] - mount_options = mount_split[2] if len(mount_split) == 3 else None + mount_options = ( + mount_split[2] if len(mount_split) == 3 else None + ) if opts.o.debug: print(f"volume_name: {volume_name}") print(f"mount path: {mount_path}") @@ -151,7 +192,7 @@ def volume_mounts_for_service(parsed_pod_files, service): volume_device = client.V1VolumeMount( mount_path=mount_path, name=volume_name, - read_only="ro" == mount_options + read_only="ro" == mount_options, ) result.append(volume_device) return result @@ -165,12 +206,19 @@ def volumes_for_pod_files(parsed_pod_files, spec, app_name): volumes = parsed_pod_file["volumes"] for volume_name in volumes.keys(): if volume_name in spec.get_configmaps(): - config_map = client.V1ConfigMapVolumeSource(name=f"{app_name}-{volume_name}") + # Set defaultMode=0o755 to make scripts executable + config_map = client.V1ConfigMapVolumeSource( + name=f"{app_name}-{volume_name}", default_mode=0o755 + ) volume = client.V1Volume(name=volume_name, config_map=config_map) result.append(volume) else: - claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=f"{app_name}-{volume_name}") - volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim) + claim = client.V1PersistentVolumeClaimVolumeSource( + claim_name=f"{app_name}-{volume_name}" + ) + volume = client.V1Volume( + name=volume_name, persistent_volume_claim=claim + ) result.append(volume) return result @@ -202,7 +250,8 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context): if "volumes" in service_obj: volumes = service_obj["volumes"] for mount_string in volumes: - # Looks like: test-data:/data or test-data:/data:ro or test-data:/data:rw + # Looks like: test-data:/data + # or test-data:/data:ro or test-data:/data:rw if opts.o.debug: print(f"mount_string: {mount_string}") mount_split = mount_string.split(":") @@ -214,15 +263,21 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context): print(f"mount path: {mount_path}") if volume_name not in deployment_context.spec.get_configmaps(): if volume_host_path_map[volume_name]: + host_path = _make_absolute_host_path( + volume_host_path_map[volume_name], + deployment_dir, + ) + container_path = get_kind_pv_bind_mount_path( + volume_name + ) volume_definitions.append( - f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n" - f" containerPath: {get_kind_pv_bind_mount_path(volume_name)}\n" + f" - hostPath: {host_path}\n" + f" containerPath: {container_path}\n" ) return ( - "" if len(volume_definitions) == 0 else ( - " extraMounts:\n" - f"{''.join(volume_definitions)}" - ) + "" + if len(volume_definitions) == 0 + else (" extraMounts:\n" f"{''.join(volume_definitions)}") ) @@ -240,25 +295,233 @@ def _generate_kind_port_mappings_from_services(parsed_pod_files): for port_string in ports: # TODO handle the complex cases # Looks like: 80 or something more complicated - port_definitions.append(f" - containerPort: {port_string}\n hostPort: {port_string}\n") + port_definitions.append( + f" - containerPort: {port_string}\n" + f" hostPort: {port_string}\n" + ) return ( - "" if len(port_definitions) == 0 else ( - " extraPortMappings:\n" - f"{''.join(port_definitions)}" - ) + "" + if len(port_definitions) == 0 + else (" extraPortMappings:\n" f"{''.join(port_definitions)}") ) def _generate_kind_port_mappings(parsed_pod_files): port_definitions = [] - # For now we just map port 80 for the nginx ingress controller we install in kind - port_string = "80" - port_definitions.append(f" - containerPort: {port_string}\n hostPort: {port_string}\n") - return ( - "" if len(port_definitions) == 0 else ( - " extraPortMappings:\n" - f"{''.join(port_definitions)}" + # Map port 80 and 443 for the Caddy ingress controller (HTTPS support) + for port_string in ["80", "443"]: + port_definitions.append( + f" - containerPort: {port_string}\n hostPort: {port_string}\n" ) + return ( + "" + if len(port_definitions) == 0 + else (" extraPortMappings:\n" f"{''.join(port_definitions)}") + ) + + +def _generate_high_memlock_spec_mount(deployment_dir: Path): + """Generate the extraMount entry for high-memlock-spec.json. + + The spec file must be mounted at the same path inside the kind node + as it appears on the host, because containerd's base_runtime_spec + references an absolute path. + """ + spec_path = deployment_dir.joinpath(constants.high_memlock_spec_filename).resolve() + return f" - hostPath: {spec_path}\n" f" containerPath: {spec_path}\n" + + +def generate_high_memlock_spec_json(): + """Generate OCI spec JSON with unlimited RLIMIT_MEMLOCK. + + This is needed for workloads like Solana validators that require large + amounts of locked memory for memory-mapped files during snapshot decompression. + + The IPC_LOCK capability alone doesn't raise the RLIMIT_MEMLOCK limit - it only + allows mlock() calls. We need to set the rlimit in the OCI runtime spec. + + IMPORTANT: This must be a complete OCI runtime spec, not just the rlimits + section. The spec is based on kind's default cri-base.json with rlimits added. + """ + import json + + # Use maximum 64-bit signed integer value for unlimited + max_rlimit = 9223372036854775807 + # Based on kind's /etc/containerd/cri-base.json with rlimits added + spec = { + "ociVersion": "1.1.0-rc.1", + "process": { + "user": {"uid": 0, "gid": 0}, + "cwd": "/", + "capabilities": { + "bounding": [ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_FSETID", + "CAP_FOWNER", + "CAP_MKNOD", + "CAP_NET_RAW", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETFCAP", + "CAP_SETPCAP", + "CAP_NET_BIND_SERVICE", + "CAP_SYS_CHROOT", + "CAP_KILL", + "CAP_AUDIT_WRITE", + ], + "effective": [ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_FSETID", + "CAP_FOWNER", + "CAP_MKNOD", + "CAP_NET_RAW", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETFCAP", + "CAP_SETPCAP", + "CAP_NET_BIND_SERVICE", + "CAP_SYS_CHROOT", + "CAP_KILL", + "CAP_AUDIT_WRITE", + ], + "permitted": [ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_FSETID", + "CAP_FOWNER", + "CAP_MKNOD", + "CAP_NET_RAW", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETFCAP", + "CAP_SETPCAP", + "CAP_NET_BIND_SERVICE", + "CAP_SYS_CHROOT", + "CAP_KILL", + "CAP_AUDIT_WRITE", + ], + }, + "rlimits": [ + {"type": "RLIMIT_MEMLOCK", "hard": max_rlimit, "soft": max_rlimit}, + {"type": "RLIMIT_NOFILE", "hard": 1048576, "soft": 1048576}, + ], + "noNewPrivileges": True, + }, + "root": {"path": "rootfs"}, + "mounts": [ + { + "destination": "/proc", + "type": "proc", + "source": "proc", + "options": ["nosuid", "noexec", "nodev"], + }, + { + "destination": "/dev", + "type": "tmpfs", + "source": "tmpfs", + "options": ["nosuid", "strictatime", "mode=755", "size=65536k"], + }, + { + "destination": "/dev/pts", + "type": "devpts", + "source": "devpts", + "options": [ + "nosuid", + "noexec", + "newinstance", + "ptmxmode=0666", + "mode=0620", + "gid=5", + ], + }, + { + "destination": "/dev/shm", + "type": "tmpfs", + "source": "shm", + "options": ["nosuid", "noexec", "nodev", "mode=1777", "size=65536k"], + }, + { + "destination": "/dev/mqueue", + "type": "mqueue", + "source": "mqueue", + "options": ["nosuid", "noexec", "nodev"], + }, + { + "destination": "/sys", + "type": "sysfs", + "source": "sysfs", + "options": ["nosuid", "noexec", "nodev", "ro"], + }, + { + "destination": "/run", + "type": "tmpfs", + "source": "tmpfs", + "options": ["nosuid", "strictatime", "mode=755", "size=65536k"], + }, + ], + "linux": { + "resources": {"devices": [{"allow": False, "access": "rwm"}]}, + "cgroupsPath": "/default", + "namespaces": [ + {"type": "pid"}, + {"type": "ipc"}, + {"type": "uts"}, + {"type": "mount"}, + {"type": "network"}, + ], + "maskedPaths": [ + "/proc/acpi", + "/proc/asound", + "/proc/kcore", + "/proc/keys", + "/proc/latency_stats", + "/proc/timer_list", + "/proc/timer_stats", + "/proc/sched_debug", + "/sys/firmware", + "/proc/scsi", + ], + "readonlyPaths": [ + "/proc/bus", + "/proc/fs", + "/proc/irq", + "/proc/sys", + "/proc/sysrq-trigger", + ], + }, + "hooks": {"createContainer": [{"path": "/kind/bin/mount-product-files.sh"}]}, + } + return json.dumps(spec, indent=2) + + +# Keep old name as alias for backward compatibility +def generate_cri_base_json(): + """Deprecated: Use generate_high_memlock_spec_json() instead.""" + return generate_high_memlock_spec_json() + + +def _generate_containerd_config_patches( + deployment_dir: Path, has_high_memlock: bool +) -> str: + """Generate containerdConfigPatches YAML for custom runtime handlers. + + This configures containerd to have a runtime handler named 'high-memlock' + that uses a custom OCI base spec with unlimited RLIMIT_MEMLOCK. + """ + if not has_high_memlock: + return "" + + spec_path = deployment_dir.joinpath(constants.high_memlock_spec_filename).resolve() + runtime_name = constants.high_memlock_runtime + plugin_path = 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes' + return ( + "containerdConfigPatches:\n" + " - |-\n" + f" [{plugin_path}.{runtime_name}]\n" + ' runtime_type = "io.containerd.runc.v2"\n' + f' base_runtime_spec = "{spec_path}"\n' ) @@ -268,28 +531,45 @@ def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]: return result -def _expand_shell_vars(raw_val: str) -> str: - # could be: or ${} or ${:-} - # TODO: implement support for variable substitution and default values - # if raw_val is like ${} print a warning and substitute an empty string - # otherwise return raw_val - match = re.search(r"^\$\{(.*)\}$", raw_val) +def _expand_shell_vars( + raw_val: str, env_map: Optional[Mapping[str, str]] = None +) -> str: + # Expand docker-compose style variable substitution: + # ${VAR} - use VAR value or empty string + # ${VAR:-default} - use VAR value or default if unset/empty + # ${VAR-default} - use VAR value or default if unset + if env_map is None: + env_map = {} + if raw_val is None: + return "" + match = re.search(r"^\$\{([^}]+)\}$", raw_val) if match: - print(f"WARNING: found unimplemented environment variable substitution: {raw_val}") - else: - return raw_val + inner = match.group(1) + # Check for default value syntax + if ":-" in inner: + var_name, default_val = inner.split(":-", 1) + return env_map.get(var_name, "") or default_val + elif "-" in inner: + var_name, default_val = inner.split("-", 1) + return env_map.get(var_name, default_val) + else: + return env_map.get(inner, "") + return raw_val -# TODO: handle the case where the same env var is defined in multiple places -def envs_from_compose_file(compose_file_envs: Mapping[str, str]) -> Mapping[str, str]: +def envs_from_compose_file( + compose_file_envs: Mapping[str, str], env_map: Optional[Mapping[str, str]] = None +) -> Mapping[str, str]: result = {} for env_var, env_val in compose_file_envs.items(): - expanded_env_val = _expand_shell_vars(env_val) + expanded_env_val = _expand_shell_vars(env_val, env_map) result.update({env_var: expanded_env_val}) return result -def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]: +def envs_from_environment_variables_map( + map: Mapping[str, str] +) -> List[client.V1EnvVar]: result = [] for env_var, env_val in map.items(): result.append(client.V1EnvVar(env_var, env_val)) @@ -320,10 +600,34 @@ def generate_kind_config(deployment_dir: Path, deployment_context): pod_files = [p for p in compose_file_dir.iterdir() if p.is_file()] parsed_pod_files_map = parsed_pod_files_map_from_file_names(pod_files) port_mappings_yml = _generate_kind_port_mappings(parsed_pod_files_map) - mounts_yml = _generate_kind_mounts(parsed_pod_files_map, deployment_dir, deployment_context) - return ( - "kind: Cluster\n" - "apiVersion: kind.x-k8s.io/v1alpha4\n" + mounts_yml = _generate_kind_mounts( + parsed_pod_files_map, deployment_dir, deployment_context + ) + + # Check if unlimited_memlock is enabled + unlimited_memlock = deployment_context.spec.get_unlimited_memlock() + + # Generate containerdConfigPatches for RuntimeClass support + containerd_patches_yml = _generate_containerd_config_patches( + deployment_dir, unlimited_memlock + ) + + # Add high-memlock spec file mount if needed + if unlimited_memlock: + spec_mount = _generate_high_memlock_spec_mount(deployment_dir) + if mounts_yml: + # Append to existing mounts + mounts_yml = mounts_yml.rstrip() + "\n" + spec_mount + else: + mounts_yml = f" extraMounts:\n{spec_mount}" + + # Build the config - containerdConfigPatches must be at cluster level (before nodes) + config = "kind: Cluster\n" "apiVersion: kind.x-k8s.io/v1alpha4\n" + + if containerd_patches_yml: + config += containerd_patches_yml + + config += ( "nodes:\n" "- role: control-plane\n" " kubeadmConfigPatches:\n" @@ -331,7 +635,9 @@ def generate_kind_config(deployment_dir: Path, deployment_context): " kind: InitConfiguration\n" " nodeRegistration:\n" " kubeletExtraArgs:\n" - " node-labels: \"ingress-ready=true\"\n" + ' node-labels: "ingress-ready=true"\n' f"{port_mappings_yml}\n" f"{mounts_yml}\n" ) + + return config diff --git a/stack_orchestrator/deploy/k8s/k8s_command.py b/stack_orchestrator/deploy/k8s/k8s_command.py new file mode 100644 index 00000000..76bc76b8 --- /dev/null +++ b/stack_orchestrator/deploy/k8s/k8s_command.py @@ -0,0 +1,43 @@ +# Copyright © 2024 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from stack_orchestrator.deploy.k8s.helpers import get_kind_cluster + + +@click.group() +@click.pass_context +def command(ctx): + """k8s cluster management commands""" + pass + + +@command.group() +@click.pass_context +def list(ctx): + """list k8s resources""" + pass + + +@list.command() +@click.pass_context +def cluster(ctx): + """Show the existing kind cluster""" + existing_cluster = get_kind_cluster() + if existing_cluster: + print(existing_cluster) + else: + print("No cluster found") diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index bc1247eb..1713f28a 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -14,6 +14,7 @@ # along with this program. If not, see . import typing +from typing import Optional import humanfriendly from pathlib import Path @@ -23,9 +24,9 @@ from stack_orchestrator import constants class ResourceLimits: - cpus: float = None - memory: int = None - storage: int = None + cpus: Optional[float] = None + memory: Optional[int] = None + storage: Optional[int] = None def __init__(self, obj=None): if obj is None: @@ -49,8 +50,8 @@ class ResourceLimits: class Resources: - limits: ResourceLimits = None - reservations: ResourceLimits = None + limits: Optional[ResourceLimits] = None + reservations: Optional[ResourceLimits] = None def __init__(self, obj=None): if obj is None: @@ -72,11 +73,10 @@ class Resources: class Spec: - obj: typing.Any - file_path: Path + file_path: Optional[Path] - def __init__(self, file_path: Path = None, obj=None) -> None: + def __init__(self, file_path: Optional[Path] = None, obj=None) -> None: if obj is None: obj = {} self.file_path = file_path @@ -105,10 +105,14 @@ class Spec: return self.obj.get(constants.configmaps_key, {}) def get_container_resources(self): - return Resources(self.obj.get(constants.resources_key, {}).get("containers", {})) + return Resources( + self.obj.get(constants.resources_key, {}).get("containers", {}) + ) def get_volume_resources(self): - return Resources(self.obj.get(constants.resources_key, {}).get(constants.volumes_key, {})) + return Resources( + self.obj.get(constants.resources_key, {}).get(constants.volumes_key, {}) + ) def get_http_proxy(self): return self.obj.get(constants.network_key, {}).get(constants.http_proxy_key, []) @@ -129,17 +133,57 @@ class Spec: return self.obj.get(constants.labels_key, {}) def get_privileged(self): - return "true" == str(self.obj.get(constants.security_key, {}).get("privileged", "false")).lower() + return ( + "true" + == str( + self.obj.get(constants.security_key, {}).get("privileged", "false") + ).lower() + ) def get_capabilities(self): return self.obj.get(constants.security_key, {}).get("capabilities", []) + def get_unlimited_memlock(self): + return ( + "true" + == str( + self.obj.get(constants.security_key, {}).get( + constants.unlimited_memlock_key, "false" + ) + ).lower() + ) + + def get_runtime_class(self): + """Get runtime class name from spec, or derive from security settings. + + The runtime class determines which containerd runtime handler to use, + allowing different pods to have different rlimit profiles (e.g., for + unlimited RLIMIT_MEMLOCK). + + Returns: + Runtime class name string, or None to use default runtime. + """ + # Explicit runtime class takes precedence + explicit = self.obj.get(constants.security_key, {}).get( + constants.runtime_class_key, None + ) + if explicit: + return explicit + + # Auto-derive from unlimited-memlock setting + if self.get_unlimited_memlock(): + return constants.high_memlock_runtime + + return None # Use default runtime + def get_deployment_type(self): return self.obj.get(constants.deploy_to_key) def is_kubernetes_deployment(self): - return self.get_deployment_type() in [constants.k8s_kind_deploy_type, - constants.k8s_deploy_type] + return self.get_deployment_type() in [ + constants.k8s_kind_deploy_type, + constants.k8s_deploy_type, + ] def is_kind_deployment(self): return self.get_deployment_type() in [constants.k8s_kind_deploy_type] diff --git a/stack_orchestrator/deploy/stack.py b/stack_orchestrator/deploy/stack.py index 39ad0083..75d40705 100644 --- a/stack_orchestrator/deploy/stack.py +++ b/stack_orchestrator/deploy/stack.py @@ -19,7 +19,6 @@ from stack_orchestrator.util import get_yaml class Stack: - name: str obj: typing.Any diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp.py b/stack_orchestrator/deploy/webapp/deploy_webapp.py index 0353ad16..6d5ea6c2 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp.py @@ -27,7 +27,9 @@ from stack_orchestrator.deploy.deploy_types import DeployCommandContext def _fixup_container_tag(deployment_dir: str, image: str): deployment_dir_path = Path(deployment_dir) - compose_file = deployment_dir_path.joinpath("compose", "docker-compose-webapp-template.yml") + compose_file = deployment_dir_path.joinpath( + "compose", "docker-compose-webapp-template.yml" + ) # replace "cerc/webapp-container:local" in the file with our image tag with open(compose_file) as rfile: contents = rfile.read() @@ -39,13 +41,13 @@ def _fixup_container_tag(deployment_dir: str, image: str): def _fixup_url_spec(spec_file_name: str, url: str): # url is like: https://example.com/path parsed_url = urlparse(url) - http_proxy_spec = f''' + http_proxy_spec = f""" http-proxy: - host-name: {parsed_url.hostname} routes: - path: '{parsed_url.path if parsed_url.path else "/"}' proxy-to: webapp:80 - ''' + """ spec_file_path = Path(spec_file_name) with open(spec_file_path) as rfile: contents = rfile.read() @@ -54,11 +56,15 @@ def _fixup_url_spec(spec_file_name: str, url: str): wfile.write(contents) -def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): +def create_deployment( + ctx, deployment_dir, image, url, kube_config, image_registry, env_file +): # Do the equivalent of: - # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml + # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init \ + # --output webapp-spec.yml # --config (eqivalent of the contents of my-config.env) - # 2. laconic-so --stack webapp-template deploy --deploy-to k8s create --deployment-dir test-deployment + # 2. laconic-so --stack webapp-template deploy --deploy-to k8s create \ + # --deployment-dir test-deployment # --spec-file webapp-spec.yml # 3. Replace the container image tag with the specified image deployment_dir_path = Path(deployment_dir) @@ -83,15 +89,12 @@ def create_deployment(ctx, deployment_dir, image, url, kube_config, image_regist kube_config, image_registry, spec_file_name, - None + None, ) # Add the TLS and DNS spec _fixup_url_spec(spec_file_name, url) create_operation( - deploy_command_context, - spec_file_name, - deployment_dir, - None, + deploy_command_context, spec_file_name, deployment_dir, False, None, None ) # Fix up the container tag inside the deployment compose file _fixup_container_tag(deployment_dir, image) @@ -101,7 +104,7 @@ def create_deployment(ctx, deployment_dir, image, url, kube_config, image_regist @click.group() @click.pass_context def command(ctx): - '''manage a webapp deployment''' + """manage a webapp deployment""" # Check that --stack wasn't supplied if ctx.parent.obj.stack: @@ -110,13 +113,20 @@ def command(ctx): @command.command() @click.option("--kube-config", help="Provide a config file for a k8s deployment") -@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") -@click.option("--deployment-dir", help="Create deployment files in this directory", required=True) +@click.option( + "--image-registry", + help="Provide a container image registry url for this k8s cluster", +) +@click.option( + "--deployment-dir", help="Create deployment files in this directory", required=True +) @click.option("--image", help="image to deploy", required=True) @click.option("--url", help="url to serve", required=True) @click.option("--env-file", help="environment file for webapp") @click.pass_context def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): - '''create a deployment for the specified webapp container''' + """create a deployment for the specified webapp container""" - return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file) + return create_deployment( + ctx, deployment_dir, image, url, kube_config, image_registry, env_file + ) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 24a529c2..92458c47 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -73,6 +73,7 @@ def process_app_deployment_request( app = laconic.get_record( app_deployment_request.attributes.application, require=True ) + assert app is not None # require=True ensures this logger.log(f"Retrieved app record {app_deployment_request.attributes.application}") # 2. determine dns @@ -112,7 +113,8 @@ def process_app_deployment_request( ) elif "preexisting" == fqdn_policy: raise Exception( - f"No pre-existing DnsRecord {dns_lrn} could be found for request {app_deployment_request.id}." + f"No pre-existing DnsRecord {dns_lrn} could be found for " + f"request {app_deployment_request.id}." ) # 4. get build and runtime config from request @@ -128,7 +130,8 @@ def process_app_deployment_request( parsed = AttrDict(yaml.safe_load(decrypted.data)) if record_owner not in parsed.authorized: raise Exception( - f"{record_owner} not authorized to access config {app_deployment_request.attributes.config.ref}" + f"{record_owner} not authorized to access config " + f"{app_deployment_request.attributes.config.ref}" ) if "env" in parsed.config: env.update(parsed.config.env) @@ -156,8 +159,10 @@ def process_app_deployment_request( deployment_record = laconic.get_record(app_deployment_lrn) deployment_dir = os.path.join(deployment_parent_dir, fqdn) - # At present we use this to generate a unique but stable ID for the app's host container - # TODO: implement support to derive this transparently from the already-unique deployment id + # At present we use this to generate a unique but stable ID for the + # app's host container + # TODO: implement support to derive this transparently from the + # already-unique deployment id unique_deployment_id = hashlib.md5(fqdn.encode()).hexdigest()[:16] deployment_config_file = os.path.join(deployment_dir, "config.env") deployment_container_tag = "laconic-webapp/%s:local" % unique_deployment_id @@ -166,11 +171,12 @@ def process_app_deployment_request( if not os.path.exists(deployment_dir): if deployment_record: raise Exception( - "Deployment record %s exists, but not deployment dir %s. Please remove name." - % (app_deployment_lrn, deployment_dir) + "Deployment record %s exists, but not deployment dir %s. " + "Please remove name." % (app_deployment_lrn, deployment_dir) ) logger.log( - f"Creating webapp deployment in: {deployment_dir} with container id: {deployment_container_tag}" + f"Creating webapp deployment in: {deployment_dir} " + f"with container id: {deployment_container_tag}" ) deploy_webapp.create_deployment( ctx, @@ -187,7 +193,8 @@ def process_app_deployment_request( needs_k8s_deploy = False if force_rebuild: logger.log( - "--force-rebuild is enabled so the container will always be built now, even if nothing has changed in the app" + "--force-rebuild is enabled so the container will always be " + "built now, even if nothing has changed in the app" ) # 6. build container (if needed) # TODO: add a comment that explains what this code is doing (not clear to me) @@ -199,11 +206,12 @@ def process_app_deployment_request( needs_k8s_deploy = True # check if the image already exists shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag) - # Note: in the code below, calls to add_tags_to_image() won't work at present. - # This is because SO deployment code in general re-names the container image - # to be unique to the deployment. This is done transparently - # and so when we call add_tags_to_image() here and try to add tags to the remote image, - # we get the image name wrong. Accordingly I've disabled the relevant code for now. + # Note: in the code below, calls to add_tags_to_image() won't + # work at present. This is because SO deployment code in general + # re-names the container image to be unique to the deployment. + # This is done transparently and so when we call add_tags_to_image() + # here and try to add tags to the remote image, we get the image + # name wrong. Accordingly I've disabled the relevant code for now. # This is safe because we are running with --force-rebuild at present if shared_tag_exists and not force_rebuild: # simply add our unique tag to the existing image and we are done @@ -211,7 +219,9 @@ def process_app_deployment_request( f"(SKIPPED) Existing image found for this app: {app_image_shared_tag} " "tagging it with: {deployment_container_tag} to use in this deployment" ) - # add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag) + # add_tags_to_image( + # image_registry, app_image_shared_tag, deployment_container_tag + # ) logger.log("Tag complete") else: extra_build_args = [] # TODO: pull from request @@ -223,11 +233,15 @@ def process_app_deployment_request( logger.log(f"Pushing container image: {deployment_container_tag}") push_container_image(deployment_dir, logger) logger.log("Push complete") - # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag. + # The build/push commands above will use the unique deployment + # tag, so now we need to add the shared tag. logger.log( - f"(SKIPPED) Adding global app image tag: {app_image_shared_tag} to newly built image: {deployment_container_tag}" + f"(SKIPPED) Adding global app image tag: {app_image_shared_tag} " + f"to newly built image: {deployment_container_tag}" ) - # add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag) + # add_tags_to_image( + # image_registry, deployment_container_tag, app_image_shared_tag + # ) logger.log("Tag complete") else: logger.log("Requested app is already deployed, skipping build and image push") @@ -306,7 +320,11 @@ def dump_known_requests(filename, requests, status="SEEN"): help="How to handle requests with an FQDN: prohibit, allow, preexisting", default="prohibit", ) -@click.option("--ip", help="IP address of the k8s deployment (to be set in DNS record)", default=None) +@click.option( + "--ip", + help="IP address of the k8s deployment (to be set in DNS record)", + default=None, +) @click.option("--record-namespace-dns", help="eg, lrn://laconic/dns", required=True) @click.option( "--record-namespace-deployments", @@ -364,7 +382,9 @@ def dump_known_requests(filename, requests, status="SEEN"): "--private-key-file", help="The private key for decrypting config.", required=True ) @click.option( - "--registry-lock-file", help="File path to use for registry mutex lock", default=None + "--registry-lock-file", + help="File path to use for registry mutex lock", + default=None, ) @click.option( "--private-key-passphrase", @@ -421,7 +441,8 @@ def command( # noqa: C901 or not dns_suffix ): print( - "--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", + "--dns-suffix, --record-namespace-dns, and " + "--record-namespace-deployments are all required", file=sys.stderr, ) sys.exit(2) @@ -459,19 +480,25 @@ def command( # noqa: C901 include_tags = [tag.strip() for tag in include_tags.split(",") if tag] exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag] - laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file) + laconic = LaconicRegistryClient( + laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file + ) webapp_deployer_record = laconic.get_record(lrn, require=True) + assert webapp_deployer_record is not None # require=True ensures this + assert webapp_deployer_record.attributes is not None payment_address = webapp_deployer_record.attributes.paymentAddress main_logger.log(f"Payment address: {payment_address}") if min_required_payment and not payment_address: print( - f"Minimum payment required, but no payment address listed for deployer: {lrn}.", + f"Minimum payment required, but no payment address listed " + f"for deployer: {lrn}.", file=sys.stderr, ) sys.exit(2) # Find deployment requests. + requests = [] # single request if request_id: main_logger.log(f"Retrieving request {request_id}...") @@ -495,25 +522,35 @@ def command( # noqa: C901 previous_requests = load_known_requests(state_file) # Collapse related requests. - requests.sort(key=lambda r: r.createTime) - requests.reverse() + # Filter out None values and sort + valid_requests = [r for r in requests if r is not None] + valid_requests.sort(key=lambda r: r.createTime if r else "") + valid_requests.reverse() requests_by_name = {} skipped_by_name = {} - for r in requests: - main_logger.log(f"BEGIN: Examining request {r.id}") + for r in valid_requests: + if not r: + continue + r_id = r.id if r else "unknown" + main_logger.log(f"BEGIN: Examining request {r_id}") result = "PENDING" try: if ( - r.id in previous_requests - and previous_requests[r.id].get("status", "") != "RETRY" + r_id in previous_requests + and previous_requests[r_id].get("status", "") != "RETRY" ): - main_logger.log(f"Skipping request {r.id}, we've already seen it.") + main_logger.log(f"Skipping request {r_id}, we've already seen it.") result = "SKIP" continue + if not r.attributes: + main_logger.log(f"Skipping request {r_id}, no attributes.") + result = "ERROR" + continue + app = laconic.get_record(r.attributes.application) if not app: - main_logger.log(f"Skipping request {r.id}, cannot locate app.") + main_logger.log(f"Skipping request {r_id}, cannot locate app.") result = "ERROR" continue @@ -521,7 +558,7 @@ def command( # noqa: C901 if not requested_name: requested_name = generate_hostname_for_app(app) main_logger.log( - "Generating name %s for request %s." % (requested_name, r.id) + "Generating name %s for request %s." % (requested_name, r_id) ) if ( @@ -529,30 +566,33 @@ def command( # noqa: C901 or requested_name in requests_by_name ): main_logger.log( - "Ignoring request %s, it has been superseded." % r.id + "Ignoring request %s, it has been superseded." % r_id ) result = "SKIP" continue if skip_by_tag(r, include_tags, exclude_tags): + r_tags = r.attributes.tags if r.attributes else None main_logger.log( - "Skipping request %s, filtered by tag (include %s, exclude %s, present %s)" - % (r.id, include_tags, exclude_tags, r.attributes.tags) + "Skipping request %s, filtered by tag " + "(include %s, exclude %s, present %s)" + % (r_id, include_tags, exclude_tags, r_tags) ) skipped_by_name[requested_name] = r result = "SKIP" continue + r_app = r.attributes.application if r.attributes else "unknown" main_logger.log( "Found pending request %s to run application %s on %s." - % (r.id, r.attributes.application, requested_name) + % (r_id, r_app, requested_name) ) requests_by_name[requested_name] = r except Exception as e: result = "ERROR" - main_logger.log(f"ERROR examining request {r.id}: " + str(e)) + main_logger.log(f"ERROR examining request {r_id}: " + str(e)) finally: - main_logger.log(f"DONE Examining request {r.id} with result {result}.") + main_logger.log(f"DONE Examining request {r_id} with result {result}.") if result in ["ERROR"]: dump_known_requests(state_file, [r], status=result) @@ -581,11 +621,13 @@ def command( # noqa: C901 cancellation_requests[r.id], r ): main_logger.log( - f"Found deployment cancellation request for {r.id} at {cancellation_requests[r.id].id}" + f"Found deployment cancellation request for {r.id} " + f"at {cancellation_requests[r.id].id}" ) elif r.id in deployments_by_request: main_logger.log( - f"Found satisfied request for {r.id} at {deployments_by_request[r.id].id}" + f"Found satisfied request for {r.id} " + f"at {deployments_by_request[r.id].id}" ) else: if ( @@ -593,7 +635,8 @@ def command( # noqa: C901 and previous_requests[r.id].get("status", "") != "RETRY" ): main_logger.log( - f"Skipping unsatisfied request {r.id} because we have seen it before." + f"Skipping unsatisfied request {r.id} " + "because we have seen it before." ) else: main_logger.log(f"Request {r.id} needs to processed.") @@ -603,13 +646,7 @@ def command( # noqa: C901 for r in requests_to_check_for_payment: if r.attributes.auction: if auction_requests: - if confirm_auction( - laconic, - r, - lrn, - payment_address, - main_logger - ): + if confirm_auction(laconic, r, lrn, payment_address, main_logger): main_logger.log(f"{r.id}: Auction confirmed.") requests_to_execute.append(r) else: @@ -652,8 +689,12 @@ def command( # noqa: C901 status = "ERROR" run_log_file = None run_reg_client = laconic + build_logger = None try: - run_id = f"{r.id}-{str(time.time()).split('.')[0]}-{str(uuid.uuid4()).split('-')[0]}" + run_id = ( + f"{r.id}-{str(time.time()).split('.')[0]}-" + f"{str(uuid.uuid4()).split('-')[0]}" + ) if log_dir: run_log_dir = os.path.join(log_dir, r.id) if not os.path.exists(run_log_dir): @@ -664,7 +705,9 @@ def command( # noqa: C901 ) run_log_file = open(run_log_file_path, "wt") run_reg_client = LaconicRegistryClient( - laconic_config, log_file=run_log_file, mutex_lock_file=registry_lock_file + laconic_config, + log_file=run_log_file, + mutex_lock_file=registry_lock_file, ) build_logger = TimedLogger(run_id, run_log_file) @@ -692,7 +735,8 @@ def command( # noqa: C901 status = "DEPLOYED" except Exception as e: main_logger.log(f"ERROR {r.id}:" + str(e)) - build_logger.log("ERROR: " + str(e)) + if build_logger: + build_logger.log("ERROR: " + str(e)) finally: main_logger.log(f"DEPLOYING {r.id}: END - {status}") if build_logger: diff --git a/stack_orchestrator/deploy/webapp/handle_deployment_auction.py b/stack_orchestrator/deploy/webapp/handle_deployment_auction.py index 0a3c65c0..933de899 100644 --- a/stack_orchestrator/deploy/webapp/handle_deployment_auction.py +++ b/stack_orchestrator/deploy/webapp/handle_deployment_auction.py @@ -44,19 +44,27 @@ def process_app_deployment_auction( # Check auction kind if auction.kind != AUCTION_KIND_PROVIDER: - raise Exception(f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction.kind}") + raise Exception( + f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction.kind}" + ) if current_status == "PENDING": # Skip if pending auction not in commit state if auction.status != AuctionStatus.COMMIT: - logger.log(f"Skipping pending request, auction {auction_id} status: {auction.status}") + logger.log( + f"Skipping pending request, auction {auction_id} " + f"status: {auction.status}" + ) return "SKIP", "" # Check max_price bid_amount_int = int(bid_amount) max_price_int = int(auction.maxPrice.quantity) if max_price_int < bid_amount_int: - logger.log(f"Skipping auction {auction_id} with max_price ({max_price_int}) less than bid_amount ({bid_amount_int})") + logger.log( + f"Skipping auction {auction_id} with max_price ({max_price_int}) " + f"less than bid_amount ({bid_amount_int})" + ) return "SKIP", "" # Bid on the auction @@ -121,7 +129,9 @@ def dump_known_auction_requests(filename, requests, status="SEEN"): required=True, ) @click.option( - "--registry-lock-file", help="File path to use for registry mutex lock", default=None + "--registry-lock-file", + help="File path to use for registry mutex lock", + default=None, ) @click.option( "--dry-run", help="Don't do anything, just report what would be done.", is_flag=True @@ -142,7 +152,9 @@ def command( logger = TimedLogger(file=sys.stderr) try: - laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file) + laconic = LaconicRegistryClient( + laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file + ) auctions_requests = laconic.app_deployment_auctions() previous_requests = {} @@ -164,7 +176,8 @@ def command( # Handle already seen requests if r.id in previous_requests: - # If it's not in commit or reveal status, skip the request as we've already seen it + # If it's not in commit or reveal status, skip the request as we've + # already seen it current_status = previous_requests[r.id].get("status", "") result_status = current_status if current_status not in ["COMMIT", "REVEAL"]: @@ -172,7 +185,10 @@ def command( continue reveal_file_path = previous_requests[r.id].get("revealFile", "") - logger.log(f"Found existing auction request {r.id} for application {application}, status {current_status}.") + logger.log( + f"Found existing auction request {r.id} for application " + f"{application}, status {current_status}." + ) else: # It's a fresh request, check application record app = laconic.get_record(application) @@ -181,7 +197,10 @@ def command( result_status = "ERROR" continue - logger.log(f"Found pending auction request {r.id} for application {application}.") + logger.log( + f"Found pending auction request {r.id} for application " + f"{application}." + ) # Add requests to be processed requests_to_execute.append((r, result_status, reveal_file_path)) @@ -190,9 +209,15 @@ def command( result_status = "ERROR" logger.log(f"ERROR: examining request {r.id}: " + str(e)) finally: - logger.log(f"DONE: Examining request {r.id} with result {result_status}.") + logger.log( + f"DONE: Examining request {r.id} with result {result_status}." + ) if result_status in ["ERROR"]: - dump_known_auction_requests(state_file, [AttrDict({"id": r.id, "revealFile": reveal_file_path})], result_status) + dump_known_auction_requests( + state_file, + [AttrDict({"id": r.id, "revealFile": reveal_file_path})], + result_status, + ) logger.log(f"Found {len(requests_to_execute)} request(s) to process.") @@ -214,7 +239,11 @@ def command( logger.log(f"ERROR {r.id}:" + str(e)) finally: logger.log(f"Processing {r.id}: END - {result_status}") - dump_known_auction_requests(state_file, [AttrDict({"id": r.id, "revealFile": reveal_file_path})], result_status) + dump_known_auction_requests( + state_file, + [AttrDict({"id": r.id, "revealFile": reveal_file_path})], + result_status, + ) except Exception as e: logger.log("UNCAUGHT ERROR:" + str(e)) raise e diff --git a/stack_orchestrator/deploy/webapp/publish_webapp_deployer.py b/stack_orchestrator/deploy/webapp/publish_webapp_deployer.py index 851e90e1..f69a2031 100644 --- a/stack_orchestrator/deploy/webapp/publish_webapp_deployer.py +++ b/stack_orchestrator/deploy/webapp/publish_webapp_deployer.py @@ -64,7 +64,11 @@ def command( # noqa: C901 ): laconic = LaconicRegistryClient(laconic_config) if not payment_address: - payment_address = laconic.whoami().address + whoami_result = laconic.whoami() + if whoami_result and whoami_result.address: + payment_address = whoami_result.address + else: + raise ValueError("Could not determine payment address from laconic whoami") pub_key = base64.b64encode(open(public_key_file, "rb").read()).decode("ASCII") hostname = urlparse(api_url).hostname diff --git a/stack_orchestrator/deploy/webapp/registry_mutex.py b/stack_orchestrator/deploy/webapp/registry_mutex.py index e464f58d..1d023230 100644 --- a/stack_orchestrator/deploy/webapp/registry_mutex.py +++ b/stack_orchestrator/deploy/webapp/registry_mutex.py @@ -17,7 +17,7 @@ def acquire_lock(client, lock_file_path, timeout): try: # Check if lock file exists and is potentially stale if os.path.exists(lock_file_path): - with open(lock_file_path, 'r') as lock_file: + with open(lock_file_path, "r") as lock_file: timestamp = float(lock_file.read().strip()) # If lock is stale, remove the lock file @@ -25,13 +25,15 @@ def acquire_lock(client, lock_file_path, timeout): print(f"Stale lock detected, removing lock file {lock_file_path}") os.remove(lock_file_path) else: - print(f"Lock file {lock_file_path} exists and is recent, waiting...") + print( + f"Lock file {lock_file_path} exists and is recent, waiting..." + ) time.sleep(LOCK_RETRY_INTERVAL) continue # Try to create a new lock file with the current timestamp fd = os.open(lock_file_path, os.O_CREAT | os.O_EXCL | os.O_RDWR) - with os.fdopen(fd, 'w') as lock_file: + with os.fdopen(fd, "w") as lock_file: lock_file.write(str(time.time())) client.mutex_lock_acquired = True diff --git a/stack_orchestrator/deploy/webapp/request_webapp_deployment.py b/stack_orchestrator/deploy/webapp/request_webapp_deployment.py index 0fb2cff1..8f266cb4 100644 --- a/stack_orchestrator/deploy/webapp/request_webapp_deployment.py +++ b/stack_orchestrator/deploy/webapp/request_webapp_deployment.py @@ -16,6 +16,7 @@ import shutil import sys import tempfile from datetime import datetime +from typing import NoReturn import base64 import gnupg @@ -31,7 +32,7 @@ from stack_orchestrator.deploy.webapp.util import ( from dotenv import dotenv_values -def fatal(msg: str): +def fatal(msg: str) -> NoReturn: print(msg, file=sys.stderr) sys.exit(1) @@ -57,7 +58,10 @@ def fatal(msg: str): @click.option("--config-ref", help="The ref of an existing config upload to use.") @click.option( "--make-payment", - help="The payment to make (in alnt). The value should be a number or 'auto' to use the deployer's minimum required payment.", + help=( + "The payment to make (in alnt). The value should be a number or " + "'auto' to use the deployer's minimum required payment." + ), ) @click.option( "--use-payment", help="The TX id of an existing, unused payment", default=None @@ -91,7 +95,10 @@ def command( # noqa: C901 sys.exit(2) if auction_id and (make_payment or use_payment): - print("Cannot specify --auction-id with --make-payment or --use-payment", file=sys.stderr) + print( + "Cannot specify --auction-id with --make-payment or --use-payment", + file=sys.stderr, + ) sys.exit(2) if env_file and config_ref: @@ -117,7 +124,10 @@ def command( # noqa: C901 # Cross check app against application in the auction record auction_app = auction_records_by_id[0].attributes.application if auction_app != app: - fatal(f"Requested application {app} does not match application from auction record {auction_app}") + fatal( + f"Requested application {app} does not match application " + f"from auction record {auction_app}" + ) # Fetch auction details auction = laconic.get_auction(auction_id) @@ -125,29 +135,42 @@ def command( # noqa: C901 fatal(f"Unable to locate auction: {auction_id}") # Check auction owner - if auction.ownerAddress != laconic.whoami().address: + whoami = laconic.whoami() + if not whoami or not whoami.address: + fatal("Unable to determine current account address") + if auction.ownerAddress != whoami.address: fatal(f"Auction {auction_id} owner mismatch") # Check auction kind - if auction.kind != AUCTION_KIND_PROVIDER: - fatal(f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction.kind}") + auction_kind = auction.kind if auction else None + if auction_kind != AUCTION_KIND_PROVIDER: + fatal( + f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction_kind}" + ) # Check auction status - if auction.status != AuctionStatus.COMPLETED: - fatal(f"Auction {auction_id} not completed yet, status {auction.status}") + auction_status = auction.status if auction else None + if auction_status != AuctionStatus.COMPLETED: + fatal(f"Auction {auction_id} not completed yet, status {auction_status}") # Check that winner list is not empty - if len(auction.winnerAddresses) == 0: + winner_addresses = auction.winnerAddresses if auction else [] + if not winner_addresses or len(winner_addresses) == 0: fatal(f"Auction {auction_id} has no winners") - auction_winners = auction.winnerAddresses + auction_winners = winner_addresses # Get deployer record for all the auction winners for auction_winner in auction_winners: # TODO: Match auction winner address with provider address? - deployer_records_by_owner = laconic.webapp_deployers({"paymentAddress": auction_winner}) + deployer_records_by_owner = laconic.webapp_deployers( + {"paymentAddress": auction_winner} + ) if len(deployer_records_by_owner) == 0: - print(f"WARNING: Unable to locate deployer for auction winner {auction_winner}") + print( + f"WARNING: Unable to locate deployer for auction winner " + f"{auction_winner}" + ) # Take first record with name set target_deployer_record = deployer_records_by_owner[0] @@ -182,9 +205,12 @@ def command( # noqa: C901 recip = gpg.list_keys()[0]["uids"][0] # Wrap the config + whoami_result = laconic.whoami() + if not whoami_result or not whoami_result.address: + fatal("Unable to determine current account address") config = { # Include account (and payment?) details - "authorized": [laconic.whoami().address], + "authorized": [whoami_result.address], "config": {"env": dict(dotenv_values(env_file))}, } serialized = yaml.dump(config) @@ -211,12 +237,22 @@ def command( # noqa: C901 if (not deployer) and len(deployer_record.names): target_deployer = deployer_record.names[0] + app_name = ( + app_record.attributes.name + if app_record and app_record.attributes + else "unknown" + ) + app_version = ( + app_record.attributes.version + if app_record and app_record.attributes + else "unknown" + ) deployment_request = { "record": { "type": "ApplicationDeploymentRequest", "application": app, "version": "1.0.0", - "name": f"{app_record.attributes.name}@{app_record.attributes.version}", + "name": f"{app_name}@{app_version}", "deployer": target_deployer, "meta": {"when": str(datetime.utcnow())}, } diff --git a/stack_orchestrator/deploy/webapp/request_webapp_undeployment.py b/stack_orchestrator/deploy/webapp/request_webapp_undeployment.py index 80cee3ce..54bf2393 100644 --- a/stack_orchestrator/deploy/webapp/request_webapp_undeployment.py +++ b/stack_orchestrator/deploy/webapp/request_webapp_undeployment.py @@ -17,12 +17,12 @@ import sys import click import yaml -from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient) +from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient -def fatal(msg: str): +def fatal(msg: str) -> None: print(msg, file=sys.stderr) - sys.exit(1) + sys.exit(1) # noqa: This function never returns @click.command() @@ -30,18 +30,19 @@ def fatal(msg: str): "--laconic-config", help="Provide a config file for laconicd", required=True ) @click.option( - "--deployer", - help="The LRN of the deployer to process this request.", - required=True + "--deployer", help="The LRN of the deployer to process this request.", required=True ) @click.option( "--deployment", - help="Deployment record (ApplicationDeploymentRecord) id of the deployment to remove.", + help="Deployment record (ApplicationDeploymentRecord) id of the deployment.", required=True, ) @click.option( "--make-payment", - help="The payment to make (in alnt). The value should be a number or 'auto' to use the deployer's minimum required payment.", + help=( + "The payment to make (in alnt). The value should be a number or " + "'auto' to use the deployer's minimum required payment." + ), ) @click.option( "--use-payment", help="The TX id of an existing, unused payment", default=None @@ -84,18 +85,17 @@ def command( if dry_run: undeployment_request["record"]["payment"] = "DRY_RUN" elif "auto" == make_payment: - if "minimumPayment" in deployer_record.attributes: - amount = int( - deployer_record.attributes.minimumPayment.replace("alnt", "") - ) + attrs = deployer_record.attributes if deployer_record else None + if attrs and "minimumPayment" in attrs: + amount = int(attrs.minimumPayment.replace("alnt", "")) else: amount = make_payment if amount: - receipt = laconic.send_tokens( - deployer_record.attributes.paymentAddress, amount - ) - undeployment_request["record"]["payment"] = receipt.tx.hash - print("Payment TX:", receipt.tx.hash) + attrs = deployer_record.attributes if deployer_record else None + if attrs and attrs.paymentAddress: + receipt = laconic.send_tokens(attrs.paymentAddress, amount) + undeployment_request["record"]["payment"] = receipt.tx.hash + print("Payment TX:", receipt.tx.hash) elif use_payment: undeployment_request["record"]["payment"] = use_payment diff --git a/stack_orchestrator/deploy/webapp/run_webapp.py b/stack_orchestrator/deploy/webapp/run_webapp.py index f780c6f8..fe11fc30 100644 --- a/stack_orchestrator/deploy/webapp/run_webapp.py +++ b/stack_orchestrator/deploy/webapp/run_webapp.py @@ -18,7 +18,8 @@ # env vars: # CERC_REPO_BASE_DIR defaults to ~/cerc -# TODO: display the available list of containers; allow re-build of either all or specific containers +# TODO: display the available list of containers; allow re-build of either +# all or specific containers import hashlib import click @@ -36,30 +37,62 @@ WEBAPP_PORT = 80 @click.option("--port", help="port to use (default random)") @click.pass_context def command(ctx, image, env_file, port): - '''run the specified webapp container''' + """run the specified webapp container""" - env = {} + env: dict[str, str] = {} if env_file: - env = dotenv_values(env_file) + # Filter out None values from dotenv + for k, v in dotenv_values(env_file).items(): + if v is not None: + env[k] = v unique_cluster_descriptor = f"{image},{env}" hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() cluster = f"laconic-webapp-{hash}" - deployer = getDeployer(type=constants.compose_deploy_type, - deployment_context=None, - compose_files=None, - compose_project_name=cluster, - compose_env_file=None) + deployer = getDeployer( + type=constants.compose_deploy_type, + deployment_context=None, + compose_files=None, + compose_project_name=cluster, + compose_env_file=None, + ) + + if not deployer: + print("Failed to create deployer", file=click.get_text_stream("stderr")) + ctx.exit(1) + return # Unreachable, but helps type checker ports = [] if port: ports = [(port, WEBAPP_PORT)] - container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, ports=ports, detach=True) + container = deployer.run( + image, + command=[], + user=None, + volumes=[], + entrypoint=None, + env=env, + ports=ports, + detach=True, + ) # Make configurable? webappPort = f"{WEBAPP_PORT}/tcp" # TODO: This assumes a Docker container object... - if webappPort in container.network_settings.ports: + # Check if container has network_settings (Docker container object) + if ( + container + and hasattr(container, "network_settings") + and container.network_settings + and hasattr(container.network_settings, "ports") + and container.network_settings.ports + and webappPort in container.network_settings.ports + ): mapping = container.network_settings.ports[webappPort][0] - print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""") + container_id = getattr(container, "id", "unknown") + print( + f"Image: {image}\n" + f"ID: {container_id}\n" + f"URL: http://localhost:{mapping['HostPort']}" + ) diff --git a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py index 90e62197..30b6eaac 100644 --- a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py @@ -43,7 +43,13 @@ def process_app_removal_request( deployment_record = laconic.get_record( app_removal_request.attributes.deployment, require=True ) + assert deployment_record is not None # require=True ensures this + assert deployment_record.attributes is not None + dns_record = laconic.get_record(deployment_record.attributes.dns, require=True) + assert dns_record is not None # require=True ensures this + assert dns_record.attributes is not None + deployment_dir = os.path.join( deployment_parent_dir, dns_record.attributes.name.lower() ) @@ -51,40 +57,50 @@ def process_app_removal_request( if not os.path.exists(deployment_dir): raise Exception("Deployment directory %s does not exist." % deployment_dir) - # Check if the removal request is from the owner of the DnsRecord or deployment record. + # Check if the removal request is from the owner of the DnsRecord or + # deployment record. matched_owner = match_owner(app_removal_request, deployment_record, dns_record) # Or of the original deployment request. if not matched_owner and deployment_record.attributes.request: - matched_owner = match_owner( - app_removal_request, - laconic.get_record(deployment_record.attributes.request, require=True), + original_request = laconic.get_record( + deployment_record.attributes.request, require=True ) + assert original_request is not None # require=True ensures this + matched_owner = match_owner(app_removal_request, original_request) if matched_owner: - main_logger.log("Matched deployment ownership:", matched_owner) + main_logger.log(f"Matched deployment ownership: {matched_owner}") else: + deployment_id = deployment_record.id if deployment_record else "unknown" + request_id = app_removal_request.id if app_removal_request else "unknown" raise Exception( "Unable to confirm ownership of deployment %s for removal request %s" - % (deployment_record.id, app_removal_request.id) + % (deployment_id, request_id) ) - # TODO(telackey): Call the function directly. The easiest way to build the correct click context is to - # exec the process, but it would be better to refactor so we could just call down_operation with the - # necessary parameters + # TODO(telackey): Call the function directly. The easiest way to build + # the correct click context is to exec the process, but it would be better + # to refactor so we could just call down_operation with the necessary + # parameters down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"] if delete_volumes: down_command.append("--delete-volumes") result = subprocess.run(down_command) result.check_returncode() + deployer_name = ( + webapp_deployer_record.names[0] + if webapp_deployer_record and webapp_deployer_record.names + else "" + ) removal_record = { "record": { "type": "ApplicationDeploymentRemovalRecord", "version": "1.0.0", - "request": app_removal_request.id, - "deployment": deployment_record.id, - "deployer": webapp_deployer_record.names[0], + "request": app_removal_request.id if app_removal_request else "", + "deployment": deployment_record.id if deployment_record else "", + "deployer": deployer_name, } } @@ -94,11 +110,11 @@ def process_app_removal_request( laconic.publish(removal_record) if delete_names: - if deployment_record.names: + if deployment_record and deployment_record.names: for name in deployment_record.names: laconic.delete_name(name) - if dns_record.names: + if dns_record and dns_record.names: for name in dns_record.names: laconic.delete_name(name) @@ -179,7 +195,9 @@ def dump_known_requests(filename, requests): is_flag=True, ) @click.option( - "--registry-lock-file", help="File path to use for registry mutex lock", default=None + "--registry-lock-file", + help="File path to use for registry mutex lock", + default=None, ) @click.pass_context def command( # noqa: C901 @@ -216,19 +234,25 @@ def command( # noqa: C901 include_tags = [tag.strip() for tag in include_tags.split(",") if tag] exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag] - laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file) + laconic = LaconicRegistryClient( + laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file + ) deployer_record = laconic.get_record(lrn, require=True) + assert deployer_record is not None # require=True ensures this + assert deployer_record.attributes is not None payment_address = deployer_record.attributes.paymentAddress main_logger.log(f"Payment address: {payment_address}") if min_required_payment and not payment_address: print( - f"Minimum payment required, but no payment address listed for deployer: {lrn}.", + f"Minimum payment required, but no payment address listed " + f"for deployer: {lrn}.", file=sys.stderr, ) sys.exit(2) # Find deployment removal requests. + requests = [] # single request if request_id: main_logger.log(f"Retrieving request {request_id}...") @@ -252,32 +276,39 @@ def command( # noqa: C901 main_logger.log(f"Loading known requests from {state_file}...") previous_requests = load_known_requests(state_file) - requests.sort(key=lambda r: r.createTime) - requests.reverse() + # Filter out None values and sort by createTime + valid_requests = [r for r in requests if r is not None] + valid_requests.sort(key=lambda r: r.createTime if r else "") + valid_requests.reverse() # Find deployments. named_deployments = {} main_logger.log("Discovering app deployments...") for d in laconic.app_deployments(all=False): - named_deployments[d.id] = d + if d and d.id: + named_deployments[d.id] = d # Find removal requests. removals_by_deployment = {} removals_by_request = {} main_logger.log("Discovering deployment removals...") for r in laconic.app_deployment_removals(): - if r.attributes.deployment: + if r and r.attributes and r.attributes.deployment: # TODO: should we handle CRNs? removals_by_deployment[r.attributes.deployment] = r one_per_deployment = {} - for r in requests: + for r in valid_requests: + if not r or not r.attributes: + continue if not r.attributes.deployment: + r_id = r.id if r else "unknown" main_logger.log( - f"Skipping removal request {r.id} since it was a cancellation." + f"Skipping removal request {r_id} since it was a cancellation." ) elif r.attributes.deployment in one_per_deployment: - main_logger.log(f"Skipping removal request {r.id} since it was superseded.") + r_id = r.id if r else "unknown" + main_logger.log(f"Skipping removal request {r_id} since it was superseded.") else: one_per_deployment[r.attributes.deployment] = r @@ -286,21 +317,25 @@ def command( # noqa: C901 try: if r.attributes.deployment not in named_deployments: main_logger.log( - f"Skipping removal request {r.id} for {r.attributes.deployment} because it does" - f"not appear to refer to a live, named deployment." + f"Skipping removal request {r.id} for " + f"{r.attributes.deployment} because it does not appear to " + "refer to a live, named deployment." ) elif skip_by_tag(r, include_tags, exclude_tags): main_logger.log( - "Skipping removal request %s, filtered by tag (include %s, exclude %s, present %s)" + "Skipping removal request %s, filtered by tag " + "(include %s, exclude %s, present %s)" % (r.id, include_tags, exclude_tags, r.attributes.tags) ) elif r.id in removals_by_request: main_logger.log( - f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}" + f"Found satisfied request for {r.id} " + f"at {removals_by_request[r.id].id}" ) elif r.attributes.deployment in removals_by_deployment: main_logger.log( - f"Found removal record for indicated deployment {r.attributes.deployment} at " + f"Found removal record for indicated deployment " + f"{r.attributes.deployment} at " f"{removals_by_deployment[r.attributes.deployment].id}" ) else: @@ -309,7 +344,8 @@ def command( # noqa: C901 requests_to_check_for_payment.append(r) else: main_logger.log( - f"Skipping unsatisfied request {r.id} because we have seen it before." + f"Skipping unsatisfied request {r.id} " + "because we have seen it before." ) except Exception as e: main_logger.log(f"ERROR examining {r.id}: {e}") diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index 991dd249..3c536477 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -25,6 +25,7 @@ import uuid import yaml from enum import Enum +from typing import Any, List, Optional, TextIO from stack_orchestrator.deploy.webapp.registry_mutex import registry_mutex @@ -41,27 +42,35 @@ AUCTION_KIND_PROVIDER = "provider" class AttrDict(dict): - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super(AttrDict, self).__init__(*args, **kwargs) self.__dict__ = self - def __getattribute__(self, attr): + def __getattribute__(self, attr: str) -> Any: __dict__ = super(AttrDict, self).__getattribute__("__dict__") if attr in __dict__: v = super(AttrDict, self).__getattribute__(attr) if isinstance(v, dict): return AttrDict(v) return v + return super(AttrDict, self).__getattribute__(attr) + + def __getattr__(self, attr: str) -> Any: + # This method is called when attribute is not found + # Return None for missing attributes (matches original behavior) + return None class TimedLogger: - def __init__(self, id="", file=None): + def __init__(self, id: str = "", file: Optional[TextIO] = None) -> None: self.start = datetime.datetime.now() self.last = self.start self.id = id self.file = file - def log(self, msg, show_step_time=True, show_total_time=False): + def log( + self, msg: str, show_step_time: bool = True, show_total_time: bool = False + ) -> None: prefix = f"{datetime.datetime.utcnow()} - {self.id}" if show_step_time: prefix += f" - {datetime.datetime.now() - self.last} (step)" @@ -79,7 +88,7 @@ def load_known_requests(filename): return {} -def logged_cmd(log_file, *vargs): +def logged_cmd(log_file: Optional[TextIO], *vargs: str) -> str: result = None try: if log_file: @@ -88,17 +97,22 @@ def logged_cmd(log_file, *vargs): result.check_returncode() return result.stdout.decode() except Exception as err: - if result: - print(result.stderr.decode(), file=log_file) - else: - print(str(err), file=log_file) + if log_file: + if result: + print(result.stderr.decode(), file=log_file) + else: + print(str(err), file=log_file) raise err -def match_owner(recordA, *records): +def match_owner( + recordA: Optional[AttrDict], *records: Optional[AttrDict] +) -> Optional[str]: + if not recordA or not recordA.owners: + return None for owner in recordA.owners: for otherRecord in records: - if owner in otherRecord.owners: + if otherRecord and otherRecord.owners and owner in otherRecord.owners: return owner return None @@ -226,25 +240,27 @@ class LaconicRegistryClient: ] # Most recent records first - results.sort(key=lambda r: r.createTime) + results.sort(key=lambda r: r.createTime or "") results.reverse() self._add_to_cache(results) return results - def _add_to_cache(self, records): + def _add_to_cache(self, records: List[AttrDict]) -> None: if not records: return for p in records: - self.cache["name_or_id"][p.id] = p + if p.id: + self.cache["name_or_id"][p.id] = p if p.names: for lrn in p.names: self.cache["name_or_id"][lrn] = p if p.attributes and p.attributes.type: - if p.attributes.type not in self.cache: - self.cache[p.attributes.type] = [] - self.cache[p.attributes.type].append(p) + attr_type = p.attributes.type + if attr_type not in self.cache: + self.cache[attr_type] = [] + self.cache[attr_type].append(p) def resolve(self, name): if not name: @@ -497,7 +513,7 @@ class LaconicRegistryClient: "--max-price", str(auction["max_price"]), "--num-providers", - str(auction["num_providers"]) + str(auction["num_providers"]), ] return json.loads(logged_cmd(self.log_file, *args))["auctionId"] @@ -556,24 +572,36 @@ def determine_base_container(clone_dir, app_type="webapp"): return base_container -def build_container_image(app_record, tag, extra_build_args=None, logger=None): +def build_container_image( + app_record: Optional[AttrDict], + tag: str, + extra_build_args: Optional[List[str]] = None, + logger: Optional[TimedLogger] = None, +) -> None: + if app_record is None: + raise ValueError("app_record cannot be None") if extra_build_args is None: extra_build_args = [] tmpdir = tempfile.mkdtemp() - # TODO: determine if this code could be calling into the Python git library like setup-repositories + # TODO: determine if this code could be calling into the Python git + # library like setup-repositories + log_file = logger.file if logger else None try: record_id = app_record["id"] ref = app_record.attributes.repository_ref repo = random.choice(app_record.attributes.repository) clone_dir = os.path.join(tmpdir, record_id) - logger.log(f"Cloning repository {repo} to {clone_dir} ...") + if logger: + logger.log(f"Cloning repository {repo} to {clone_dir} ...") # Set github credentials if present running a command like: - # git config --global url."https://${TOKEN}:@github.com/".insteadOf "https://github.com/" + # git config --global url."https://${TOKEN}:@github.com/".insteadOf + # "https://github.com/" github_token = os.environ.get("DEPLOYER_GITHUB_TOKEN") if github_token: - logger.log("Github token detected, setting it in the git environment") + if logger: + logger.log("Github token detected, setting it in the git environment") git_config_args = [ "git", "config", @@ -581,9 +609,7 @@ def build_container_image(app_record, tag, extra_build_args=None, logger=None): f"url.https://{github_token}:@github.com/.insteadOf", "https://github.com/", ] - result = subprocess.run( - git_config_args, stdout=logger.file, stderr=logger.file - ) + result = subprocess.run(git_config_args, stdout=log_file, stderr=log_file) result.check_returncode() if ref: # TODO: Determing branch or hash, and use depth 1 if we can. @@ -594,29 +620,32 @@ def build_container_image(app_record, tag, extra_build_args=None, logger=None): subprocess.check_call( ["git", "clone", repo, clone_dir], env=git_env, - stdout=logger.file, - stderr=logger.file, + stdout=log_file, + stderr=log_file, ) except Exception as e: - logger.log(f"git clone failed. Is the repository {repo} private?") + if logger: + logger.log(f"git clone failed. Is the repository {repo} private?") raise e try: subprocess.check_call( ["git", "checkout", ref], cwd=clone_dir, env=git_env, - stdout=logger.file, - stderr=logger.file, + stdout=log_file, + stderr=log_file, ) except Exception as e: - logger.log(f"git checkout failed. Does ref {ref} exist?") + if logger: + logger.log(f"git checkout failed. Does ref {ref} exist?") raise e else: - # TODO: why is this code different vs the branch above (run vs check_call, and no prompt disable)? + # TODO: why is this code different vs the branch above (run vs check_call, + # and no prompt disable)? result = subprocess.run( ["git", "clone", "--depth", "1", repo, clone_dir], - stdout=logger.file, - stderr=logger.file, + stdout=log_file, + stderr=log_file, ) result.check_returncode() @@ -624,7 +653,8 @@ def build_container_image(app_record, tag, extra_build_args=None, logger=None): clone_dir, app_record.attributes.app_type ) - logger.log("Building webapp ...") + if logger: + logger.log("Building webapp ...") build_command = [ sys.argv[0], "--verbose", @@ -640,10 +670,10 @@ def build_container_image(app_record, tag, extra_build_args=None, logger=None): build_command.append("--extra-build-args") build_command.append(" ".join(extra_build_args)) - result = subprocess.run(build_command, stdout=logger.file, stderr=logger.file) + result = subprocess.run(build_command, stdout=log_file, stderr=log_file) result.check_returncode() finally: - logged_cmd(logger.file, "rm", "-rf", tmpdir) + logged_cmd(log_file, "rm", "-rf", tmpdir) def push_container_image(deployment_dir, logger): @@ -749,9 +779,13 @@ def publish_deployment( # Set auction or payment id from request if app_deployment_request.attributes.auction: - new_deployment_record["record"]["auction"] = app_deployment_request.attributes.auction + new_deployment_record["record"][ + "auction" + ] = app_deployment_request.attributes.auction elif app_deployment_request.attributes.payment: - new_deployment_record["record"]["payment"] = app_deployment_request.attributes.payment + new_deployment_record["record"][ + "payment" + ] = app_deployment_request.attributes.payment if webapp_deployer_record: new_deployment_record["record"]["deployer"] = webapp_deployer_record.names[0] @@ -801,7 +835,13 @@ def skip_by_tag(r, include_tags, exclude_tags): return False -def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min_amount, logger): +def confirm_payment( + laconic: LaconicRegistryClient, + record: AttrDict, + payment_address: str, + min_amount: int, + logger: TimedLogger, +) -> bool: req_owner = laconic.get_owner(record) if req_owner == payment_address: # No need to confirm payment if the sender and recipient are the same account. @@ -818,31 +858,35 @@ def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min if tx.code != 0: logger.log( - f"{record.id}: payment tx {tx.hash} was not successful - code: {tx.code}, log: {tx.log}" + f"{record.id}: payment tx {tx.hash} was not successful - " + f"code: {tx.code}, log: {tx.log}" ) return False if tx.sender != req_owner: logger.log( - f"{record.id}: payment sender {tx.sender} in tx {tx.hash} does not match deployment " - f"request owner {req_owner}" + f"{record.id}: payment sender {tx.sender} in tx {tx.hash} " + f"does not match deployment request owner {req_owner}" ) return False if tx.recipient != payment_address: logger.log( - f"{record.id}: payment recipient {tx.recipient} in tx {tx.hash} does not match {payment_address}" + f"{record.id}: payment recipient {tx.recipient} in tx {tx.hash} " + f"does not match {payment_address}" ) return False - pay_denom = "".join([i for i in tx.amount if not i.isdigit()]) + tx_amount = tx.amount or "" + pay_denom = "".join([i for i in tx_amount if not i.isdigit()]) if pay_denom != "alnt": logger.log( - f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected payment denomination" + f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected " + "payment denomination" ) return False - pay_amount = int("".join([i for i in tx.amount if i.isdigit()])) + pay_amount = int("".join([i for i in tx_amount if i.isdigit()]) or "0") if pay_amount < min_amount: logger.log( f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}" @@ -858,8 +902,12 @@ def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min used_request = laconic.get_record(used[0].attributes.request, require=True) # Check that payment was used for deployment of same application - if record.attributes.application != used_request.attributes.application: - logger.log(f"{record.id}: payment {tx.hash} already used on a different application deployment {used}") + used_app = used_request.attributes.application if used_request else None + if record.attributes.application != used_app: + logger.log( + f"{record.id}: payment {tx.hash} already used on a different " + f"application deployment {used}" + ) return False used = laconic.app_deployment_removals( @@ -874,7 +922,13 @@ def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min return True -def confirm_auction(laconic: LaconicRegistryClient, record, deployer_lrn, payment_address, logger): +def confirm_auction( + laconic: LaconicRegistryClient, + record: AttrDict, + deployer_lrn: str, + payment_address: str, + logger: TimedLogger, +) -> bool: auction_id = record.attributes.auction auction = laconic.get_auction(auction_id) @@ -886,11 +940,16 @@ def confirm_auction(laconic: LaconicRegistryClient, record, deployer_lrn, paymen # Cross check app against application in the auction record requested_app = laconic.get_record(record.attributes.application, require=True) - auction_app = laconic.get_record(auction_records_by_id[0].attributes.application, require=True) - if requested_app.id != auction_app.id: + auction_app = laconic.get_record( + auction_records_by_id[0].attributes.application, require=True + ) + requested_app_id = requested_app.id if requested_app else None + auction_app_id = auction_app.id if auction_app else None + if requested_app_id != auction_app_id: logger.log( - f"{record.id}: requested application {record.attributes.application} does not match application from " - f"auction record {auction_records_by_id[0].attributes.application}" + f"{record.id}: requested application {record.attributes.application} " + f"does not match application from auction record " + f"{auction_records_by_id[0].attributes.application}" ) return False diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index a50c7c9b..826ef4ff 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -21,37 +21,41 @@ from stack_orchestrator.repos import fetch_stack from stack_orchestrator.build import build_containers, fetch_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp -from stack_orchestrator.deploy.webapp import (run_webapp, - deploy_webapp, - deploy_webapp_from_registry, - undeploy_webapp_from_registry, - publish_webapp_deployer, - publish_deployment_auction, - handle_deployment_auction, - request_webapp_deployment, - request_webapp_undeployment) +from stack_orchestrator.deploy.webapp import ( + run_webapp, + deploy_webapp, + deploy_webapp_from_registry, + undeploy_webapp_from_registry, + publish_webapp_deployer, + publish_deployment_auction, + handle_deployment_auction, + request_webapp_deployment, + request_webapp_undeployment, +) from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment from stack_orchestrator import opts from stack_orchestrator import update -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @click.group(context_settings=CONTEXT_SETTINGS) -@click.option('--stack', help="specify a stack to build/deploy") -@click.option('--quiet', is_flag=True, default=False) -@click.option('--verbose', is_flag=True, default=False) -@click.option('--dry-run', is_flag=True, default=False) -@click.option('--local-stack', is_flag=True, default=False) -@click.option('--debug', is_flag=True, default=False) -@click.option('--continue-on-error', is_flag=True, default=False) +@click.option("--stack", help="specify a stack to build/deploy") +@click.option("--quiet", is_flag=True, default=False) +@click.option("--verbose", is_flag=True, default=False) +@click.option("--dry-run", is_flag=True, default=False) +@click.option("--local-stack", is_flag=True, default=False) +@click.option("--debug", is_flag=True, default=False) +@click.option("--continue-on-error", is_flag=True, default=False) # See: https://click.palletsprojects.com/en/8.1.x/complex/#building-a-git-clone @click.pass_context def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error): """Laconic Stack Orchestrator""" - command_options = CommandOptions(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error) + command_options = CommandOptions( + stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error + ) opts.opts.o = command_options ctx.obj = command_options diff --git a/stack_orchestrator/opts.py b/stack_orchestrator/opts.py index 665da535..064224dd 100644 --- a/stack_orchestrator/opts.py +++ b/stack_orchestrator/opts.py @@ -17,4 +17,4 @@ from stack_orchestrator.command_types import CommandOptions class opts: - o: CommandOptions = None + o: CommandOptions = None # type: ignore[assignment] # Set at runtime diff --git a/stack_orchestrator/repos/fetch_stack.py b/stack_orchestrator/repos/fetch_stack.py index 9566e48f..cee97d0c 100644 --- a/stack_orchestrator/repos/fetch_stack.py +++ b/stack_orchestrator/repos/fetch_stack.py @@ -29,14 +29,16 @@ from stack_orchestrator.util import error_exit @click.command() -@click.argument('stack-locator') -@click.option('--git-ssh', is_flag=True, default=False) -@click.option('--check-only', is_flag=True, default=False) -@click.option('--pull', is_flag=True, default=False) +@click.argument("stack-locator") +@click.option("--git-ssh", is_flag=True, default=False) +@click.option("--check-only", is_flag=True, default=False) +@click.option("--pull", is_flag=True, default=False) @click.pass_context def command(ctx, stack_locator, git_ssh, check_only, pull): - '''optionally resolve then git clone a repository containing one or more stack definitions''' - dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + """Optionally resolve then git clone a repository with stack definitions.""" + dev_root_path = os.path.expanduser( + str(config("CERC_REPO_BASE_DIR", default="~/cerc")) + ) if not opts.o.quiet: print(f"Dev Root is: {dev_root_path}") try: diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index 83075647..6edd8085 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -20,20 +20,26 @@ import os import sys from decouple import config import git -from git.exc import GitCommandError +from git.exc import GitCommandError, InvalidGitRepositoryError +from typing import Any from tqdm import tqdm import click import importlib.resources from stack_orchestrator.opts import opts -from stack_orchestrator.util import get_parsed_stack_config, include_exclude_check, error_exit, warn_exit +from stack_orchestrator.util import ( + get_parsed_stack_config, + include_exclude_check, + error_exit, + warn_exit, +) class GitProgress(git.RemoteProgress): def __init__(self): super().__init__() - self.pbar = tqdm(unit='B', ascii=True, unit_scale=True) + self.pbar = tqdm(unit="B", ascii=True, unit_scale=True) - def update(self, op_code, cur_count, max_count=None, message=''): + def update(self, op_code, cur_count, max_count=None, message=""): self.pbar.total = max_count self.pbar.n = cur_count self.pbar.refresh() @@ -43,17 +49,19 @@ def is_git_repo(path): try: _ = git.Repo(path).git_dir return True - except git.exc.InvalidGitRepositoryError: + except InvalidGitRepositoryError: return False + # TODO: find a place for this in the context of click # parser = argparse.ArgumentParser( -# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" +# epilog="Config provided either in .env or settings.ini or env vars: " +# "CERC_REPO_BASE_DIR (defaults to ~/cerc)" # ) def branch_strip(s): - return s.split('@')[0] + return s.split("@")[0] def host_and_path_for_repo(fully_qualified_repo): @@ -63,10 +71,14 @@ def host_and_path_for_repo(fully_qualified_repo): # Legacy unqualified repo means github if len(repo_host_split) == 2: return "github.com", "/".join(repo_host_split), repo_branch + elif len(repo_host_split) == 3: + # First part is the host + return repo_host_split[0], "/".join(repo_host_split[1:]), repo_branch else: - if len(repo_host_split) == 3: - # First part is the host - return repo_host_split[0], "/".join(repo_host_split[1:]), repo_branch + raise ValueError( + f"Invalid repository format: {fully_qualified_repo}. " + "Expected format: host/org/repo or org/repo" + ) # See: https://stackoverflow.com/questions/18659425/get-git-current-branch-tag-name @@ -74,43 +86,64 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path): current_repo_branch_or_tag = "***UNDETERMINED***" is_branch = False try: - current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).active_branch.name + current_repo_branch_or_tag = git.Repo( + full_filesystem_repo_path + ).active_branch.name is_branch = True except TypeError: # This means that the current ref is not a branch, so possibly a tag # Let's try to get the tag try: - current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match") - # Note that git is asymmetric -- the tag you told it to check out may not be the one - # you get back here (if there are multiple tags associated with the same commit) + current_repo_branch_or_tag = git.Repo( + full_filesystem_repo_path + ).git.describe("--tags", "--exact-match") + # Note that git is asymmetric -- the tag you told it to check out + # may not be the one you get back here (if there are multiple tags + # associated with the same commit) except GitCommandError: - # If there is no matching branch or tag checked out, just use the current SHA - current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).commit("HEAD").hexsha + # If there is no matching branch or tag checked out, just use the current + # SHA + current_repo_branch_or_tag = ( + git.Repo(full_filesystem_repo_path).commit("HEAD").hexsha + ) return current_repo_branch_or_tag, is_branch # TODO: fix the messy arg list here -def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo): +def process_repo( + pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo +): if opts.o.verbose: print(f"Processing repo: {fully_qualified_repo}") repo_host, repo_path, repo_branch = host_and_path_for_repo(fully_qualified_repo) git_ssh_prefix = f"git@{repo_host}:" git_http_prefix = f"https://{repo_host}/" - full_github_repo_path = f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo_path}" + full_github_repo_path = ( + f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo_path}" + ) repoName = repo_path.split("/")[-1] full_filesystem_repo_path = os.path.join(dev_root_path, repoName) is_present = os.path.isdir(full_filesystem_repo_path) - (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag( - full_filesystem_repo_path - ) if is_present else (None, None) + (current_repo_branch_or_tag, is_branch) = ( + _get_repo_current_branch_or_tag(full_filesystem_repo_path) + if is_present + else (None, None) + ) if not opts.o.quiet: - present_text = f"already exists active {'branch' if is_branch else 'ref'}: {current_repo_branch_or_tag}" if is_present \ - else 'Needs to be fetched' + present_text = ( + f"already exists active {'branch' if is_branch else 'ref'}: " + f"{current_repo_branch_or_tag}" + if is_present + else "Needs to be fetched" + ) print(f"Checking: {full_filesystem_repo_path}: {present_text}") # Quick check that it's actually a repo if is_present: if not is_git_repo(full_filesystem_repo_path): - print(f"Error: {full_filesystem_repo_path} does not contain a valid git repository") + print( + f"Error: {full_filesystem_repo_path} does not contain " + "a valid git repository" + ) sys.exit(1) else: if pull: @@ -128,11 +161,18 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully if not is_present: # Clone if opts.o.verbose: - print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}') + print( + f"Running git clone for {full_github_repo_path} " + f"into {full_filesystem_repo_path}" + ) if not opts.o.dry_run: - git.Repo.clone_from(full_github_repo_path, - full_filesystem_repo_path, - progress=None if opts.o.quiet else GitProgress()) + # Cast to Any to work around GitPython's incomplete type stubs + progress: Any = None if opts.o.quiet else GitProgress() + git.Repo.clone_from( + full_github_repo_path, + full_filesystem_repo_path, + progress=progress, + ) else: print("(git clone skipped)") # Checkout the requested branch, if one was specified @@ -150,9 +190,9 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully if branch_to_checkout: if current_repo_branch_or_tag is None or ( - current_repo_branch_or_tag and ( - current_repo_branch_or_tag != branch_to_checkout) - ): + current_repo_branch_or_tag + and (current_repo_branch_or_tag != branch_to_checkout) + ): if not opts.o.quiet: print(f"switching to branch {branch_to_checkout} in repo {repo_path}") git_repo = git.Repo(full_filesystem_repo_path) @@ -180,14 +220,14 @@ def parse_branches(branches_string): @click.command() @click.option("--include", help="only clone these repositories") -@click.option("--exclude", help="don\'t clone these repositories") -@click.option('--git-ssh', is_flag=True, default=False) -@click.option('--check-only', is_flag=True, default=False) -@click.option('--pull', is_flag=True, default=False) +@click.option("--exclude", help="don't clone these repositories") +@click.option("--git-ssh", is_flag=True, default=False) +@click.option("--check-only", is_flag=True, default=False) +@click.option("--pull", is_flag=True, default=False) @click.option("--branches", help="override branches for repositories") @click.pass_context def command(ctx, include, exclude, git_ssh, check_only, pull, branches): - '''git clone the set of repositories required to build the complete system from source''' + """git clone the set of repositories required to build the system.""" quiet = opts.o.quiet verbose = opts.o.verbose @@ -204,22 +244,30 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches): local_stack = ctx.obj.local_stack if local_stack: - dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] - print(f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}") + dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")] + print( + f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " + f"{dev_root_path}" + ) else: - dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + dev_root_path = os.path.expanduser( + str(config("CERC_REPO_BASE_DIR", default="~/cerc")) + ) if not quiet: print(f"Dev Root is: {dev_root_path}") if not os.path.isdir(dev_root_path): if not quiet: - print('Dev root directory doesn\'t exist, creating') + print("Dev root directory doesn't exist, creating") os.makedirs(dev_root_path) # See: https://stackoverflow.com/a/20885799/1701505 from stack_orchestrator import data - with importlib.resources.open_text(data, "repository-list.txt") as repository_list_file: + + with importlib.resources.open_text( + data, "repository-list.txt" + ) as repository_list_file: all_repos = repository_list_file.read().splitlines() repos_in_scope = [] @@ -247,5 +295,5 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches): for repo in repos: try: process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, repo) - except git.exc.GitCommandError as error: + except GitCommandError as error: error_exit(f"\n******* git command returned error exit status:\n{error}") diff --git a/stack_orchestrator/update.py b/stack_orchestrator/update.py index a41eabae..85fb8b41 100644 --- a/stack_orchestrator/update.py +++ b/stack_orchestrator/update.py @@ -29,7 +29,7 @@ from stack_orchestrator.util import get_yaml def _download_url(url: str, file_path: Path): r = requests.get(url, stream=True) r.raw.decode_content = True - with open(file_path, 'wb') as f: + with open(file_path, "wb") as f: shutil.copyfileobj(r.raw, f) @@ -40,12 +40,14 @@ def _error_exit(s: str): # Note at present this probably won't work on non-Unix based OSes like Windows @click.command() -@click.option("--check-only", is_flag=True, default=False, help="only check, don't update") +@click.option( + "--check-only", is_flag=True, default=False, help="only check, don't update" +) @click.pass_context def command(ctx, check_only): - '''update shiv binary from a distribution url''' + """update shiv binary from a distribution url""" # Get the distribution URL from config - config_key = 'distribution-url' + config_key = "distribution-url" config_file_path = Path(os.path.expanduser("~/.laconic-so/config.yml")) if not config_file_path.exists(): _error_exit(f"Error: Config file: {config_file_path} not found") @@ -59,7 +61,9 @@ def command(ctx, check_only): _error_exit(f"ERROR: distribution url: {distribution_url} is not valid") # Figure out the filename for ourselves shiv_binary_path = Path(sys.argv[0]) - timestamp_filename = f"laconic-so-download-{datetime.datetime.now().strftime('%y%m%d-%H%M%S')}" + timestamp_filename = ( + f"laconic-so-download-{datetime.datetime.now().strftime('%y%m%d-%H%M%S')}" + ) temp_download_path = shiv_binary_path.parent.joinpath(timestamp_filename) # Download the file to a temp filename if ctx.obj.verbose: @@ -87,4 +91,4 @@ def command(ctx, check_only): print(f"Replacing: {shiv_binary_path} with {temp_download_path}") os.replace(temp_download_path, shiv_binary_path) if not ctx.obj.quiet: - print("Run \"laconic-so version\" to see the newly installed version") + print('Run "laconic-so version" to see the newly installed version') diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index 18dbae02..fc8437ca 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -19,7 +19,7 @@ import sys import ruamel.yaml from pathlib import Path from dotenv import dotenv_values -from typing import Mapping, Set, List +from typing import Mapping, NoReturn, Optional, Set, List from stack_orchestrator.constants import stack_file_name, deployment_file_name @@ -38,8 +38,10 @@ def get_stack_path(stack): if stack_is_external(stack): stack_path = Path(stack) else: - # In order to be compatible with Python 3.8 we need to use this hack to get the path: - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + # In order to be compatible with Python 3.8 we need to use this hack + # to get the path: + # See: https://stackoverflow.com/questions/25389095/ + # python-get-path-of-root-project-structure stack_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack) return stack_path @@ -47,10 +49,15 @@ def get_stack_path(stack): def get_dev_root_path(ctx): if ctx and ctx.local_stack: # TODO: This code probably doesn't work - dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] - print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")] + print( + f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " + f"{dev_root_path}" + ) else: - dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + dev_root_path = os.path.expanduser( + str(config("CERC_REPO_BASE_DIR", default="~/cerc")) + ) return dev_root_path @@ -78,6 +85,22 @@ def get_pod_list(parsed_stack): return result +def get_job_list(parsed_stack): + # Return list of jobs from stack config, or empty list if no jobs defined + if "jobs" not in parsed_stack: + return [] + jobs = parsed_stack["jobs"] + if not jobs: + return [] + if type(jobs[0]) is str: + result = jobs + else: + result = [] + for job in jobs: + result.append(job["name"]) + return result + + def get_plugin_code_paths(stack) -> List[Path]: parsed_stack = get_parsed_stack_config(stack) pods = parsed_stack["pods"] @@ -86,7 +109,9 @@ def get_plugin_code_paths(stack) -> List[Path]: if type(pod) is str: result.add(get_stack_path(stack)) else: - pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) + pod_root_dir = os.path.join( + get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"] + ) result.add(Path(os.path.join(pod_root_dir, "stack"))) return list(result) @@ -119,25 +144,61 @@ def resolve_compose_file(stack, pod_name: str): return compose_base.joinpath(f"docker-compose-{pod_name}.yml") +# Find a job compose file in compose-jobs directory +def resolve_job_compose_file(stack, job_name: str): + if stack_is_external(stack): + # First try looking in the external stack for the job compose file + compose_jobs_base = Path(stack).parent.parent.joinpath("compose-jobs") + proposed_file = compose_jobs_base.joinpath(f"docker-compose-{job_name}.yml") + if proposed_file.exists(): + return proposed_file + # If we don't find it fall through to the internal case + # TODO: Add internal compose-jobs directory support if needed + # For now, jobs are expected to be in external stacks only + compose_jobs_base = Path(stack).parent.parent.joinpath("compose-jobs") + return compose_jobs_base.joinpath(f"docker-compose-{job_name}.yml") + + def get_pod_file_path(stack, parsed_stack, pod_name: str): pods = parsed_stack["pods"] + result = None if type(pods[0]) is str: result = resolve_compose_file(stack, pod_name) else: for pod in pods: if pod["name"] == pod_name: - pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) + pod_root_dir = os.path.join( + get_dev_root_path(None), + pod["repository"].split("/")[-1], + pod["path"], + ) result = os.path.join(pod_root_dir, "docker-compose.yml") return result +def get_job_file_path(stack, parsed_stack, job_name: str): + if "jobs" not in parsed_stack or not parsed_stack["jobs"]: + return None + jobs = parsed_stack["jobs"] + if type(jobs[0]) is str: + result = resolve_job_compose_file(stack, job_name) + else: + # TODO: Support complex job definitions if needed + result = resolve_job_compose_file(stack, job_name) + return result + + def get_pod_script_paths(parsed_stack, pod_name: str): pods = parsed_stack["pods"] result = [] if not type(pods[0]) is str: for pod in pods: if pod["name"] == pod_name: - pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) + pod_root_dir = os.path.join( + get_dev_root_path(None), + pod["repository"].split("/")[-1], + pod["path"], + ) if "pre_start_command" in pod: result.append(os.path.join(pod_root_dir, pod["pre_start_command"])) if "post_start_command" in pod: @@ -147,6 +208,7 @@ def get_pod_script_paths(parsed_stack, pod_name: str): def pod_has_scripts(parsed_stack, pod_name: str): pods = parsed_stack["pods"] + result = False if type(pods[0]) is str: result = False else: @@ -158,7 +220,8 @@ def pod_has_scripts(parsed_stack, pod_name: str): def get_internal_compose_file_dir(): # TODO: refactor to use common code with deploy command - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + # See: + # https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure data_dir = Path(__file__).absolute().parent.joinpath("data") source_compose_dir = data_dir.joinpath("compose") return source_compose_dir @@ -220,15 +283,15 @@ def global_options2(ctx): return ctx.parent.obj -def error_exit(s): +def error_exit(s) -> NoReturn: print(f"ERROR: {s}") sys.exit(1) -def warn_exit(s): +def warn_exit(s) -> NoReturn: print(f"WARN: {s}") sys.exit(0) -def env_var_map_from_file(file: Path) -> Mapping[str, str]: +def env_var_map_from_file(file: Path) -> Mapping[str, Optional[str]]: return dotenv_values(file) diff --git a/stack_orchestrator/version.py b/stack_orchestrator/version.py index 541e5580..67bb6b13 100644 --- a/stack_orchestrator/version.py +++ b/stack_orchestrator/version.py @@ -20,10 +20,11 @@ from importlib import resources, metadata @click.command() @click.pass_context def command(ctx): - '''print tool version''' + """print tool version""" # See: https://stackoverflow.com/a/20885799/1701505 from stack_orchestrator import data + if resources.is_resource(data, "build_tag.txt"): with resources.open_text(data, "build_tag.txt") as version_file: # TODO: code better version that skips comment lines diff --git a/tests/k8s-deployment-control/run-test.sh b/tests/k8s-deployment-control/run-test.sh index 8ca9064b..c6e43694 100755 --- a/tests/k8s-deployment-control/run-test.sh +++ b/tests/k8s-deployment-control/run-test.sh @@ -116,7 +116,7 @@ echo "deploy create output file test: passed" # Note we also turn up the log level on the scheduler in order to diagnose placement errors # See logs like: kubectl -n kube-system logs kube-scheduler-laconic-f185cd245d8dba98-control-plane kind_config_file=${test_deployment_dir}/kind-config.yml -cat << EOF > ${kind_config_file} +cat << EOF > ${kind_config_file} kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 kubeadmConfigPatches: diff --git a/tests/laconic-network/run-test.sh b/tests/laconic-network/run-test.sh index 47a8de03..8cdfe045 100755 --- a/tests/laconic-network/run-test.sh +++ b/tests/laconic-network/run-test.sh @@ -14,7 +14,7 @@ chain_id="laconic_81337-6" node_moniker_prefix="node" echo "Deleting any existing network directories..." -for (( i=1 ; i<=$node_count ; i++ )); +for (( i=1 ; i<=$node_count ; i++ )); do node_network_dir=${node_dir_prefix}${i} if [[ -d $node_network_dir ]]; then @@ -38,7 +38,7 @@ do done echo "Initalizing ${node_count} nodes networks..." -for (( i=1 ; i<=$node_count ; i++ )); +for (( i=1 ; i<=$node_count ; i++ )); do node_network_dir=${node_dir_prefix}${i} node_moniker=${node_moniker_prefix}${i} @@ -47,7 +47,7 @@ do done echo "Joining ${node_count} nodes to the network..." -for (( i=1 ; i<=$node_count ; i++ )); +for (( i=1 ; i<=$node_count ; i++ )); do node_network_dir=${node_dir_prefix}${i} node_moniker=${node_moniker_prefix}${i} diff --git a/tests/mainnet-eth/run-test.sh b/tests/mainnet-eth/run-test.sh index dd56c183..332d8df7 100755 --- a/tests/mainnet-eth/run-test.sh +++ b/tests/mainnet-eth/run-test.sh @@ -15,7 +15,7 @@ echo "Test version command" reported_version_string=$( $TEST_TARGET_SO version ) echo "Version reported is: ${reported_version_string}" echo "Cloning repositories into: $CERC_REPO_BASE_DIR" -$TEST_TARGET_SO --stack mainnet-eth setup-repositories +$TEST_TARGET_SO --stack mainnet-eth setup-repositories $TEST_TARGET_SO --stack mainnet-eth build-containers $TEST_TARGET_SO --stack mainnet-eth deploy init --output mainnet-eth-spec.yml $TEST_TARGET_SO deploy create --spec-file mainnet-eth-spec.yml --deployment-dir $DEPLOYMENT_DIR diff --git a/tests/webapp-test/run-webapp-test.sh b/tests/webapp-test/run-webapp-test.sh index 6a12d54c..f950334e 100755 --- a/tests/webapp-test/run-webapp-test.sh +++ b/tests/webapp-test/run-webapp-test.sh @@ -40,7 +40,7 @@ sleep 3 wget --tries 20 --retry-connrefused --waitretry=3 -O test.before -m http://localhost:3000 docker logs $CONTAINER_ID -docker remove -f $CONTAINER_ID +docker rm -f $CONTAINER_ID echo "Running app container test" CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d ${app_image_name}) @@ -48,7 +48,7 @@ sleep 3 wget --tries 20 --retry-connrefused --waitretry=3 -O test.after -m http://localhost:3000 docker logs $CONTAINER_ID -docker remove -f $CONTAINER_ID +docker rm -f $CONTAINER_ID echo "###########################################################################" echo "" diff --git a/tox.ini b/tox.ini index e261061b..239cd79b 100644 --- a/tox.ini +++ b/tox.ini @@ -3,4 +3,3 @@ extend-ignore = E203 exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,venv max-complexity = 25 max-line-length = 132 -