Compare commits

...

8 Commits

Author SHA1 Message Date
dceda936b7 env 2024-10-07 18:55:42 +00:00
bca332b598 run dockerfile if exists 2024-10-07 18:27:10 +00:00
f1fdc48aaa Work around this bug: https://github.com/python/cpython/pull/14064 (#941)
Otherwise we sometimes see errors like:

```
cerc-webapp-deployer:   File "/root/.shiv/laconic-so_0f937aa98c2748ef9af8585d6f441dbc01546ace0d6660cbb159d1e5040aeddf/site-packages/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py", line 671, in command
cerc-webapp-deployer:     shutil.rmtree(tempdir)
cerc-webapp-deployer:   File "/usr/lib/python3.10/shutil.py", line 725, in rmtree
cerc-webapp-deployer:     _rmtree_safe_fd(fd, path, onerror)
cerc-webapp-deployer:   File "/usr/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
cerc-webapp-deployer:     onerror(os.unlink, fullname, sys.exc_info())
cerc-webapp-deployer:   File "/usr/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
cerc-webapp-deployer:     os.unlink(entry.name, dir_fd=topfd)
cerc-webapp-deployer: FileNotFoundError: [Errno 2] No such file or directory: 'S.gpg-agent.extra'
```

Reviewed-on: cerc-io/stack-orchestrator#941
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>
2024-08-28 23:17:13 +00:00
a54072de6c Add --config-ref flag. (#939)
Add a flag to re-use config.

Reviewed-on: cerc-io/stack-orchestrator#939
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>
2024-08-28 17:32:52 +00:00
fa21ff2627 Support uploaded config, add 'publish-webapp-deployer' and 'request-webapp-deployment' commands (#938)
This adds two new commands: `publish-webapp-deployer` and `request-webapp-deployment`.

`publish-webapp-deployer` creates a `WebappDeployer` record, which provides information to requestors like the API URL, minimum required payment, payment address, and public key to use for encrypting config.

```
$ laconic-so publish-deployer-to-registry \
  --laconic-config ~/.laconic/laconic.yml \
  --api-url https://webapp-deployer-api.dev.vaasl.io \
  --public-key-file webapp-deployer-api.dev.vaasl.io.pgp.pub  \
  --lrn lrn://laconic/deployers/webapp-deployer-api.dev.vaasl.io  \
  --min-required-payment 100000
```

`request-webapp-deployment` simplifies publishing a `WebappDeploymentRequest` and can also handle automatic payment, and encryption and upload of configuration.

```
$ laconic-so request-webapp-deployment \
  --laconic-config ~/.laconic/laconic.yml \
  --deployer lrn://laconic/deployers/webapp-deployer-api.dev.vaasl.io \
  --app lrn://cerc-io/applications/webapp-hello-world@0.1.3 \
  --env-file ~/yaml/hello.env \
  --make-payment auto
```

Related changes are included for the deploy/undeploy commands for decrypting and using config, using the payment address from the WebappDeployer record, etc.

Reviewed-on: cerc-io/stack-orchestrator#938
2024-08-27 19:55:06 +00:00
33d395e213 Add package registry stack instructions (#937)
- The instructions to `Deploy Gitea Package Registry` from build-support [readme](https://git.vdb.to/deep-stack/stack-orchestrator/src/branch/pm-update-registry-steps/stack_orchestrator/data/stacks/build-support#2-deploy-gitea-package-registry) don't seem to be in a working state
- Updated `package-registry` stack instructions to use deployment pattern

Reviewed-on: cerc-io/stack-orchestrator#937
Reviewed-by: ashwin <ashwin@noreply.git.vdb.to>
Reviewed-by: David Boreham <dboreham@noreply.git.vdb.to>
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
2024-08-23 09:42:44 +00:00
75ff60752a Require payment for app deployment requests. (#928)
Adds three new options for deployment/undeployment:

```
    "--min-required-payment",
    help="Requests must have a minimum payment to be processed",

    "--payment-address",
    help="The address to which payments should be made.  Default is the current laconic account.",

    "--all-requests",
    help="Handle requests addressed to anyone (by default only requests to my payment address are examined).",
```

In this mode, requests should be designated for a particular address with the attribute `to` and include a `payment` attribute which is the tx hash for the payment.

The deployer will confirm the payment (to the right account, right amount, not used before, etc.) and then proceed with the deployment or undeployment.

Reviewed-on: cerc-io/stack-orchestrator#928
Reviewed-by: David Boreham <dboreham@noreply.git.vdb.to>
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>
2024-08-21 14:39:20 +00:00
44b9709717 Use Laconic version of ping-pub (#930)
Reviewed-on: cerc-io/stack-orchestrator#930
Co-authored-by: David Boreham <david@bozemanpass.com>
Co-committed-by: David Boreham <david@bozemanpass.com>
2024-08-20 17:44:00 +00:00
12 changed files with 1019 additions and 140 deletions

View File

@ -11,3 +11,5 @@ tomli==2.0.1
validators==0.22.0
kubernetes>=28.1.0
humanfriendly>=10.0
python-gnupg>=0.5.2
requests>=2.3.2

View File

@ -30,6 +30,12 @@ from stack_orchestrator.build import build_containers
from stack_orchestrator.deploy.webapp.util import determine_base_container, TimedLogger
from stack_orchestrator.build.build_types import BuildContext
def create_env_file(env_vars, repo_root):
env_file_path = os.path.join(repo_root, '.env')
with open(env_file_path, 'w') as env_file:
for key, value in env_vars.items():
env_file.write(f"{key}={value}\n")
return env_file_path
@click.command()
@click.option('--base-container')
@ -37,8 +43,9 @@ from stack_orchestrator.build.build_types import BuildContext
@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
@click.option("--extra-build-args", help="Supply extra arguments to build")
@click.option("--tag", help="Container tag (default: cerc/<app_name>:local)")
@click.option("--env", help="Environment variables for webapp (format: KEY1=VALUE1,KEY2=VALUE2)", default="")
@click.pass_context
def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag, env):
'''build the specified webapp container'''
logger = TimedLogger()
@ -88,9 +95,28 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
base_container.replace("/", "-"),
"Dockerfile.webapp")
# Check if Dockerfile exists in the repository
repo_dockerfile = os.path.join(container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"], "Dockerfile")
default_dockerfile = os.path.join(container_build_dir,
base_container.replace("/", "-"),
"Dockerfile.webapp")
if os.path.isfile(repo_dockerfile):
env_vars = {}
if env:
for pair in env.split(','):
key, value = pair.split('=')
env_vars[key.strip()] = value.strip()
container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = repo_dockerfile
# Create .env file with environment variables
env_file_path = create_env_file(env_vars, container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"])
container_build_env["CERC_CONTAINER_BUILD_ENV_FILE"] = env_file_path
else:
container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = default_dockerfile
if not tag:
webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
tag = f"cerc/{webapp_name}:local"

View File

@ -4,5 +4,9 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
# Two-stage build is to allow us to pick up both the upstream repo's files, and local files here for config
docker build -t cerc/ping-pub-base:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile.base $CERC_REPO_BASE_DIR/explorer
docker build -t cerc/ping-pub-base:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile.base $CERC_REPO_BASE_DIR/cosmos-explorer
if [[ $? -ne 0 ]]; then
echo "FATAL: Base container build failed, exiting"
exit 1
fi
docker build -t cerc/ping-pub:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile $SCRIPT_DIR

View File

@ -10,7 +10,7 @@ repos:
- git.vdb.to/cerc-io/registry-sdk
- git.vdb.to/cerc-io/laconic-registry-cli
- git.vdb.to/cerc-io/laconic-console
- github.com/ping-pub/explorer
- git.vdb.to/cerc-io/cosmos-explorer
npms:
- registry-sdk
- laconic-registry-cli

View File

@ -2,4 +2,28 @@
The Package Registry Stack supports a build environment that requires a package registry (initially for NPM packages only).
Setup instructions can be found [here](../build-support/README.md).
## Setup
* Setup required repos and build containers:
```bash
laconic-so --stack package-registry setup-repositories
laconic-so --stack package-registry build-containers
```
* Create a deployment:
```bash
laconic-so --stack package-registry deploy init --output package-registry-spec.yml
# Update port mapping in the laconic-loaded.spec file to resolve port conflicts on host if any
laconic-so --stack package-registry deploy create --deployment-dir package-registry-deployment --spec-file package-registry-spec.yml
```
* Start the deployment:
```bash
laconic-so deployment --dir package-registry-deployment start
```
* The local gitea registry can now be accessed at <http://localhost:3000> (the username and password can be taken from the deployment logs)

View File

@ -14,6 +14,7 @@
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import os
import base64
from kubernetes import client
from typing import Any, List, Set
@ -260,12 +261,12 @@ class ClusterInfo:
for f in os.listdir(cfg_map_path):
full_path = os.path.join(cfg_map_path, f)
if os.path.isfile(full_path):
data[f] = open(full_path, 'rt').read()
data[f] = base64.b64encode(open(full_path, 'rb').read()).decode('ASCII')
spec = client.V1ConfigMap(
metadata=client.V1ObjectMeta(name=f"{self.app_name}-{cfg_map_name}",
labels={"configmap-label": cfg_map_name}),
data=data
binary_data=data
)
result.append(spec)
return result

View File

@ -21,12 +21,15 @@ import sys
import tempfile
import time
import uuid
import yaml
import click
import gnupg
from stack_orchestrator.deploy.images import remote_image_exists
from stack_orchestrator.deploy.webapp import deploy_webapp
from stack_orchestrator.deploy.webapp.util import (
AttrDict,
LaconicRegistryClient,
TimedLogger,
build_container_image,
@ -38,6 +41,7 @@ from stack_orchestrator.deploy.webapp.util import (
generate_hostname_for_app,
match_owner,
skip_by_tag,
confirm_payment,
)
@ -54,6 +58,10 @@ def process_app_deployment_request(
force_rebuild,
fqdn_policy,
recreate_on_deploy,
webapp_deployer_record,
gpg,
private_key_passphrase,
config_upload_dir,
logger,
):
logger.log("BEGIN - process_app_deployment_request")
@ -78,6 +86,9 @@ def process_app_deployment_request(
else:
fqdn = f"{requested_name}.{default_dns_suffix}"
# Normalize case (just in case)
fqdn = fqdn.lower()
# 3. check ownership of existing dnsrecord vs this request
dns_lrn = f"{dns_record_namespace}/{fqdn}"
dns_record = laconic.get_record(dns_lrn)
@ -102,14 +113,31 @@ def process_app_deployment_request(
)
# 4. get build and runtime config from request
env = {}
if app_deployment_request.attributes.config:
if "ref" in app_deployment_request.attributes.config:
with open(
f"{config_upload_dir}/{app_deployment_request.attributes.config.ref}",
"rb",
) as file:
record_owner = laconic.get_owner(app_deployment_request)
decrypted = gpg.decrypt_file(file, passphrase=private_key_passphrase)
parsed = AttrDict(yaml.safe_load(decrypted.data))
if record_owner not in parsed.authorized:
raise Exception(
f"{record_owner} not authorized to access config {app_deployment_request.attributes.config.ref}"
)
if "env" in parsed.config:
env.update(parsed.config.env)
if "env" in app_deployment_request.attributes.config:
env.update(app_deployment_request.attributes.config.env)
env_filename = None
if (
app_deployment_request.attributes.config
and "env" in app_deployment_request.attributes.config
):
if env:
env_filename = tempfile.mktemp()
with open(env_filename, "w") as file:
for k, v in app_deployment_request.attributes.config["env"].items():
for k, v in env.items():
file.write("%s=%s\n" % (k, shlex.quote(str(v))))
# 5. determine new or existing deployment
@ -119,7 +147,7 @@ def process_app_deployment_request(
app_deployment_lrn = app_deployment_request.attributes.deployment
if not app_deployment_lrn.startswith(deployment_record_namespace):
raise Exception(
"Deployment CRN %s is not in a supported namespace"
"Deployment LRN %s is not in a supported namespace"
% app_deployment_request.attributes.deployment
)
@ -222,6 +250,7 @@ def process_app_deployment_request(
dns_lrn,
deployment_dir,
app_deployment_request,
webapp_deployer_record,
logger,
)
logger.log("Publication complete.")
@ -279,8 +308,12 @@ def dump_known_requests(filename, requests, status="SEEN"):
help="How to handle requests with an FQDN: prohibit, allow, preexisting",
default="prohibit",
)
@click.option("--record-namespace-dns", help="eg, lrn://laconic/dns")
@click.option("--record-namespace-deployments", help="eg, lrn://laconic/deployments")
@click.option("--record-namespace-dns", help="eg, lrn://laconic/dns", required=True)
@click.option(
"--record-namespace-deployments",
help="eg, lrn://laconic/deployments",
required=True,
)
@click.option(
"--dry-run", help="Don't do anything, just report what would be done.", is_flag=True
)
@ -305,6 +338,31 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option(
"--log-dir", help="Output build/deployment logs to directory.", default=None
)
@click.option(
"--min-required-payment",
help="Requests must have a minimum payment to be processed (in alnt)",
default=0,
)
@click.option("--lrn", help="The LRN of this deployer.", required=True)
@click.option(
"--all-requests",
help="Handle requests addressed to anyone (by default only requests to"
"my payment address are examined).",
is_flag=True,
)
@click.option(
"--config-upload-dir",
help="The directory containing uploaded config.",
required=True,
)
@click.option(
"--private-key-file", help="The private key for decrypting config.", required=True
)
@click.option(
"--private-key-passphrase",
help="The passphrase for the private key.",
required=True,
)
@click.pass_context
def command( # noqa: C901
ctx,
@ -326,6 +384,12 @@ def command( # noqa: C901
force_rebuild,
recreate_on_deploy,
log_dir,
min_required_payment,
lrn,
config_upload_dir,
private_key_file,
private_key_passphrase,
all_requests,
):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
@ -358,6 +422,18 @@ def command( # noqa: C901
)
sys.exit(2)
tempdir = tempfile.mkdtemp()
gpg = gnupg.GPG(gnupghome=tempdir)
# Import the deployer's public key
result = gpg.import_keys(open(private_key_file, "rb").read())
if 1 != result.imported:
print(
f"Failed to load private key file: {private_key_file}.",
file=sys.stderr,
)
sys.exit(2)
main_logger = TimedLogger(file=sys.stderr)
try:
@ -366,6 +442,16 @@ def command( # noqa: C901
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr)
webapp_deployer_record = laconic.get_record(lrn, require=True)
payment_address = webapp_deployer_record.attributes.paymentAddress
main_logger.log(f"Payment address: {payment_address}")
if min_required_payment and not payment_address:
print(
f"Minimum payment required, but no payment address listed for deployer: {lrn}.",
file=sys.stderr,
)
sys.exit(2)
# Find deployment requests.
# single request
@ -375,18 +461,20 @@ def command( # noqa: C901
# all requests
elif discover:
main_logger.log("Discovering deployment requests...")
requests = laconic.app_deployment_requests()
if all_requests:
requests = laconic.app_deployment_requests()
else:
requests = laconic.app_deployment_requests({"deployer": lrn})
if only_update_state:
if not dry_run:
dump_known_requests(state_file, requests)
return
previous_requests = {}
if state_file:
main_logger.log(f"Loading known requests from {state_file}...")
previous_requests = load_known_requests(state_file)
else:
previous_requests = {}
# Collapse related requests.
requests.sort(key=lambda r: r.createTime)
@ -452,7 +540,10 @@ def command( # noqa: C901
# Find deployments.
main_logger.log("Discovering existing app deployments...")
deployments = laconic.app_deployments()
if all_requests:
deployments = laconic.app_deployments()
else:
deployments = laconic.app_deployments({"deployer": lrn})
deployments_by_request = {}
for d in deployments:
if d.attributes.request:
@ -466,7 +557,7 @@ def command( # noqa: C901
if r.attributes.request:
cancellation_requests[r.attributes.request] = r
requests_to_execute = []
requests_to_check_for_payment = []
for r in requests_by_name.values():
if r.id in cancellation_requests and match_owner(
cancellation_requests[r.id], r
@ -488,7 +579,28 @@ def command( # noqa: C901
)
else:
main_logger.log(f"Request {r.id} needs to processed.")
requests_to_check_for_payment.append(r)
requests_to_execute = []
if min_required_payment:
for r in requests_to_check_for_payment:
main_logger.log(f"{r.id}: Confirming payment...")
if confirm_payment(
laconic,
r,
payment_address,
min_required_payment,
main_logger,
):
main_logger.log(f"{r.id}: Payment confirmed.")
requests_to_execute.append(r)
else:
main_logger.log(
f"Skipping request {r.id}: unable to verify payment."
)
dump_known_requests(state_file, [r], status="UNPAID")
else:
requests_to_execute = requests_to_check_for_payment
main_logger.log(
"Found %d unsatisfied request(s) to process." % len(requests_to_execute)
@ -531,6 +643,10 @@ def command( # noqa: C901
force_rebuild,
fqdn_policy,
recreate_on_deploy,
webapp_deployer_record,
gpg,
private_key_passphrase,
config_upload_dir,
build_logger,
)
status = "DEPLOYED"
@ -551,3 +667,5 @@ def command( # noqa: C901
except Exception as e:
main_logger.log("UNCAUGHT ERROR:" + str(e))
raise e
finally:
shutil.rmtree(tempdir, ignore_errors=True)

View File

@ -0,0 +1,91 @@
# Copyright ©2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import base64
import click
import sys
import yaml
from urllib.parse import urlparse
from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient
@click.command()
@click.option(
"--laconic-config", help="Provide a config file for laconicd", required=True
)
@click.option("--api-url", help="The API URL of the deployer.", required=True)
@click.option(
"--public-key-file",
help="The public key to use. This should be a binary file.",
required=True,
)
@click.option(
"--lrn", help="eg, lrn://laconic/deployers/my.deployer.name", required=True
)
@click.option(
"--payment-address",
help="The address to which payments should be made. "
"Default is the current laconic account.",
default=None,
)
@click.option(
"--min-required-payment",
help="List the minimum required payment (in alnt) to process a deployment request.",
default=0,
)
@click.option(
"--dry-run",
help="Don't publish anything, just report what would be done.",
is_flag=True,
)
@click.pass_context
def command( # noqa: C901
ctx,
laconic_config,
api_url,
public_key_file,
lrn,
payment_address,
min_required_payment,
dry_run,
):
laconic = LaconicRegistryClient(laconic_config)
if not payment_address:
payment_address = laconic.whoami().address
pub_key = base64.b64encode(open(public_key_file, "rb").read()).decode("ASCII")
hostname = urlparse(api_url).hostname
webapp_deployer_record = {
"record": {
"type": "WebappDeployer",
"version": "1.0.0",
"apiUrl": api_url,
"name": hostname,
"publicKey": pub_key,
"paymentAddress": payment_address,
}
}
if min_required_payment:
webapp_deployer_record["record"][
"minimumPayment"
] = f"{min_required_payment}alnt"
if dry_run:
yaml.dump(webapp_deployer_record, sys.stdout)
return
laconic.publish(webapp_deployer_record, [lrn])

View File

@ -0,0 +1,175 @@
# Copyright ©2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
import base64
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import shutil
import sys
import tempfile
from datetime import datetime
import gnupg
import click
import requests
import yaml
from stack_orchestrator.deploy.webapp.util import (
LaconicRegistryClient,
)
from dotenv import dotenv_values
def fatal(msg: str):
print(msg, file=sys.stderr)
sys.exit(1)
@click.command()
@click.option(
"--laconic-config", help="Provide a config file for laconicd", required=True
)
@click.option(
"--app",
help="The LRN of the application to deploy.",
required=True,
)
@click.option(
"--deployer",
help="The LRN of the deployer to process this request.",
required=True,
)
@click.option("--env-file", help="environment file for webapp")
@click.option("--config-ref", help="The ref of an existing config upload to use.")
@click.option(
"--make-payment",
help="The payment to make (in alnt). The value should be a number or 'auto' to use the deployer's minimum required payment.",
)
@click.option(
"--use-payment", help="The TX id of an existing, unused payment", default=None
)
@click.option("--dns", help="the DNS name to request (default is autogenerated)")
@click.option(
"--dry-run",
help="Don't publish anything, just report what would be done.",
is_flag=True,
)
@click.pass_context
def command(
ctx,
laconic_config,
app,
deployer,
env_file,
config_ref,
make_payment,
use_payment,
dns,
dry_run,
): # noqa: C901
tempdir = tempfile.mkdtemp()
try:
laconic = LaconicRegistryClient(laconic_config)
app_record = laconic.get_record(app)
if not app_record:
fatal(f"Unable to locate app: {app}")
deployer_record = laconic.get_record(deployer)
if not deployer_record:
fatal(f"Unable to locate deployer: {deployer}")
if env_file and config_ref:
fatal("Cannot use --env-file and --config-ref at the same time.")
# If env_file
if env_file:
gpg = gnupg.GPG(gnupghome=tempdir)
# Import the deployer's public key
result = gpg.import_keys(
base64.b64decode(deployer_record.attributes.publicKey)
)
if 1 != result.imported:
fatal("Failed to import deployer's public key.")
recip = gpg.list_keys()[0]["uids"][0]
# Wrap the config
config = {
# Include account (and payment?) details
"authorized": [laconic.whoami().address],
"config": {"env": dict(dotenv_values(env_file))},
}
serialized = yaml.dump(config)
# Encrypt
result = gpg.encrypt(serialized, recip, always_trust=True, armor=False)
if not result.ok:
fatal("Failed to encrypt config.")
# Upload it to the deployer's API
response = requests.post(
f"{deployer_record.attributes.apiUrl}/upload/config",
data=result.data,
headers={"Content-Type": "application/octet-stream"},
)
if not response.ok:
response.raise_for_status()
config_ref = response.json()["id"]
deployment_request = {
"record": {
"type": "ApplicationDeploymentRequest",
"application": app,
"version": "1.0.0",
"name": f"{app_record.attributes.name}@{app_record.attributes.version}",
"deployer": deployer,
"meta": {"when": str(datetime.utcnow())},
}
}
if config_ref:
deployment_request["record"]["config"] = {"ref": config_ref}
if dns:
deployment_request["record"]["dns"] = dns.lower()
if make_payment:
amount = 0
if dry_run:
deployment_request["record"]["payment"] = "DRY_RUN"
elif "auto" == make_payment:
if "minimumPayment" in deployer_record.attributes:
amount = int(
deployer_record.attributes.minimumPayment.replace("alnt", "")
)
else:
amount = make_payment
if amount:
receipt = laconic.send_tokens(
deployer_record.attributes.paymentAddress, amount
)
deployment_request["record"]["payment"] = receipt.tx.hash
print("Payment TX:", receipt.tx.hash)
elif use_payment:
deployment_request["record"]["payment"] = use_payment
if dry_run:
print(yaml.dump(deployment_request))
return
# Send the request
laconic.publish(deployment_request)
finally:
shutil.rmtree(tempdir, ignore_errors=True)

View File

@ -20,18 +20,33 @@ import sys
import click
from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner, skip_by_tag
from stack_orchestrator.deploy.webapp.util import (
TimedLogger,
LaconicRegistryClient,
match_owner,
skip_by_tag,
confirm_payment,
)
main_logger = TimedLogger(file=sys.stderr)
def process_app_removal_request(ctx,
laconic: LaconicRegistryClient,
app_removal_request,
deployment_parent_dir,
delete_volumes,
delete_names):
deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True)
def process_app_removal_request(
ctx,
laconic: LaconicRegistryClient,
app_removal_request,
deployment_parent_dir,
delete_volumes,
delete_names,
webapp_deployer_record,
):
deployment_record = laconic.get_record(
app_removal_request.attributes.deployment, require=True
)
dns_record = laconic.get_record(deployment_record.attributes.dns, require=True)
deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name)
deployment_dir = os.path.join(
deployment_parent_dir, dns_record.attributes.name.lower()
)
if not os.path.exists(deployment_dir):
raise Exception("Deployment directory %s does not exist." % deployment_dir)
@ -41,13 +56,18 @@ def process_app_removal_request(ctx,
# Or of the original deployment request.
if not matched_owner and deployment_record.attributes.request:
matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.attributes.request, require=True))
matched_owner = match_owner(
app_removal_request,
laconic.get_record(deployment_record.attributes.request, require=True),
)
if matched_owner:
print("Matched deployment ownership:", matched_owner)
main_logger.log("Matched deployment ownership:", matched_owner)
else:
raise Exception("Unable to confirm ownership of deployment %s for removal request %s" %
(deployment_record.id, app_removal_request.id))
raise Exception(
"Unable to confirm ownership of deployment %s for removal request %s"
% (deployment_record.id, app_removal_request.id)
)
# TODO(telackey): Call the function directly. The easiest way to build the correct click context is to
# exec the process, but it would be better to refactor so we could just call down_operation with the
@ -64,8 +84,13 @@ def process_app_removal_request(ctx,
"version": "1.0.0",
"request": app_removal_request.id,
"deployment": deployment_record.id,
"deployer": webapp_deployer_record.names[0],
}
}
if app_removal_request.attributes.payment:
removal_record["record"]["payment"] = app_removal_request.attributes.payment
laconic.publish(removal_record)
if delete_names:
@ -97,22 +122,80 @@ def dump_known_requests(filename, requests):
@click.command()
@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
@click.option(
"--laconic-config", help="Provide a config file for laconicd", required=True
)
@click.option(
"--deployment-parent-dir",
help="Create deployment directories beneath this directory",
required=True,
)
@click.option("--request-id", help="The ApplicationDeploymentRemovalRequest to process")
@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRemovalRequests",
is_flag=True, default=False)
@click.option("--state-file", help="File to store state about previously seen requests.")
@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
@click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True)
@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes")
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
@click.option(
"--discover",
help="Discover and process all pending ApplicationDeploymentRemovalRequests",
is_flag=True,
default=False,
)
@click.option(
"--state-file", help="File to store state about previously seen requests."
)
@click.option(
"--only-update-state",
help="Only update the state file, don't process any requests anything.",
is_flag=True,
)
@click.option(
"--delete-names/--preserve-names",
help="Delete all names associated with removed deployments.",
default=True,
)
@click.option(
"--delete-volumes/--preserve-volumes", default=True, help="delete data volumes"
)
@click.option(
"--dry-run", help="Don't do anything, just report what would be done.", is_flag=True
)
@click.option(
"--include-tags",
help="Only include requests with matching tags (comma-separated).",
default="",
)
@click.option(
"--exclude-tags",
help="Exclude requests with matching tags (comma-separated).",
default="",
)
@click.option(
"--min-required-payment",
help="Requests must have a minimum payment to be processed (in alnt)",
default=0,
)
@click.option("--lrn", help="The LRN of this deployer.", required=True)
@click.option(
"--all-requests",
help="Handle requests addressed to anyone (by default only requests to"
"my payment address are examined).",
is_flag=True,
)
@click.pass_context
def command(ctx, laconic_config, deployment_parent_dir,
request_id, discover, state_file, only_update_state,
delete_names, delete_volumes, dry_run, include_tags, exclude_tags):
def command( # noqa: C901
ctx,
laconic_config,
deployment_parent_dir,
request_id,
discover,
state_file,
only_update_state,
delete_names,
delete_volumes,
dry_run,
include_tags,
exclude_tags,
min_required_payment,
lrn,
all_requests,
):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
sys.exit(2)
@ -129,34 +212,55 @@ def command(ctx, laconic_config, deployment_parent_dir,
include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
laconic = LaconicRegistryClient(laconic_config)
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr)
deployer_record = laconic.get_record(lrn, require=True)
payment_address = deployer_record.attributes.paymentAddress
main_logger.log(f"Payment address: {payment_address}")
if min_required_payment and not payment_address:
print(
f"Minimum payment required, but no payment address listed for deployer: {lrn}.",
file=sys.stderr,
)
sys.exit(2)
# Find deployment removal requests.
# single request
if request_id:
main_logger.log(f"Retrieving request {request_id}...")
requests = [laconic.get_record(request_id, require=True)]
# TODO: assert record type
# all requests
elif discover:
requests = laconic.app_deployment_removal_requests()
main_logger.log("Discovering removal requests...")
if all_requests:
requests = laconic.app_deployment_removal_requests()
else:
requests = laconic.app_deployment_removal_requests({"deployer": lrn})
if only_update_state:
if not dry_run:
dump_known_requests(state_file, requests)
return
previous_requests = load_known_requests(state_file)
previous_requests = {}
if state_file:
main_logger.log(f"Loading known requests from {state_file}...")
previous_requests = load_known_requests(state_file)
requests.sort(key=lambda r: r.createTime)
requests.reverse()
# Find deployments.
deployments = {}
for d in laconic.app_deployments(all=True):
deployments[d.id] = d
named_deployments = {}
main_logger.log("Discovering app deployments...")
for d in laconic.app_deployments(all=False):
named_deployments[d.id] = d
# Find removal requests.
removals_by_deployment = {}
removals_by_request = {}
main_logger.log("Discovering deployment removals...")
for r in laconic.app_deployment_removals():
if r.attributes.deployment:
# TODO: should we handle CRNs?
@ -165,33 +269,69 @@ def command(ctx, laconic_config, deployment_parent_dir,
one_per_deployment = {}
for r in requests:
if not r.attributes.deployment:
print(f"Skipping removal request {r.id} since it was a cancellation.")
main_logger.log(
f"Skipping removal request {r.id} since it was a cancellation."
)
elif r.attributes.deployment in one_per_deployment:
print(f"Skipping removal request {r.id} since it was superseded.")
main_logger.log(f"Skipping removal request {r.id} since it was superseded.")
else:
one_per_deployment[r.attributes.deployment] = r
requests_to_execute = []
requests_to_check_for_payment = []
for r in one_per_deployment.values():
if skip_by_tag(r, include_tags, exclude_tags):
print("Skipping removal request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id,
include_tags,
exclude_tags,
r.attributes.tags))
elif r.id in removals_by_request:
print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}")
elif r.attributes.deployment in removals_by_deployment:
print(
f"Found removal record for indicated deployment {r.attributes.deployment} at "
f"{removals_by_deployment[r.attributes.deployment].id}")
else:
if r.id not in previous_requests:
print(f"Request {r.id} needs to processed.")
try:
if r.attributes.deployment not in named_deployments:
main_logger.log(
f"Skipping removal request {r.id} for {r.attributes.deployment} because it does"
f"not appear to refer to a live, named deployment."
)
elif skip_by_tag(r, include_tags, exclude_tags):
main_logger.log(
"Skipping removal request %s, filtered by tag (include %s, exclude %s, present %s)"
% (r.id, include_tags, exclude_tags, r.attributes.tags)
)
elif r.id in removals_by_request:
main_logger.log(
f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}"
)
elif r.attributes.deployment in removals_by_deployment:
main_logger.log(
f"Found removal record for indicated deployment {r.attributes.deployment} at "
f"{removals_by_deployment[r.attributes.deployment].id}"
)
else:
if r.id not in previous_requests:
main_logger.log(f"Request {r.id} needs to processed.")
requests_to_check_for_payment.append(r)
else:
main_logger.log(
f"Skipping unsatisfied request {r.id} because we have seen it before."
)
except Exception as e:
main_logger.log(f"ERROR examining {r.id}: {e}")
requests_to_execute = []
if min_required_payment:
for r in requests_to_check_for_payment:
main_logger.log(f"{r.id}: Confirming payment...")
if confirm_payment(
laconic,
r,
payment_address,
min_required_payment,
main_logger,
):
main_logger.log(f"{r.id}: Payment confirmed.")
requests_to_execute.append(r)
else:
print(f"Skipping unsatisfied request {r.id} because we have seen it before.")
main_logger.log(f"Skipping request {r.id}: unable to verify payment.")
dump_known_requests(state_file, [r])
else:
requests_to_execute = requests_to_check_for_payment
print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
main_logger.log(
"Found %d unsatisfied request(s) to process." % len(requests_to_execute)
)
if not dry_run:
for r in requests_to_execute:
@ -202,7 +342,10 @@ def command(ctx, laconic_config, deployment_parent_dir,
r,
os.path.abspath(deployment_parent_dir),
delete_volumes,
delete_names
delete_names,
deployer_record,
)
except Exception as e:
main_logger.log(f"ERROR processing removal request {r.id}: {e}")
finally:
dump_known_requests(state_file, [r])

View File

@ -1,4 +1,4 @@
# Copyright © 2023 Vulcanize
# = str(min_required_payment) Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -22,7 +22,6 @@ import subprocess
import sys
import tempfile
import uuid
import yaml
@ -83,6 +82,84 @@ def match_owner(recordA, *records):
return None
def is_lrn(name_or_id: str):
if name_or_id:
return str(name_or_id).startswith("lrn://")
return False
def is_id(name_or_id: str):
return not is_lrn(name_or_id)
def confirm_payment(laconic, record, payment_address, min_amount, logger):
req_owner = laconic.get_owner(record)
if req_owner == payment_address:
# No need to confirm payment if the sender and recipient are the same account.
return True
if not record.attributes.payment:
logger.log(f"{record.id}: no payment tx info")
return False
tx = laconic.get_tx(record.attributes.payment)
if not tx:
logger.log(f"{record.id}: cannot locate payment tx")
return False
if tx.code != 0:
logger.log(
f"{record.id}: payment tx {tx.hash} was not successful - code: {tx.code}, log: {tx.log}"
)
return False
if tx.sender != req_owner:
logger.log(
f"{record.id}: payment sender {tx.sender} in tx {tx.hash} does not match deployment "
f"request owner {req_owner}"
)
return False
if tx.recipient != payment_address:
logger.log(
f"{record.id}: payment recipient {tx.recipient} in tx {tx.hash} does not match {payment_address}"
)
return False
pay_denom = "".join([i for i in tx.amount if not i.isdigit()])
if pay_denom != "alnt":
logger.log(
f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected payment denomination"
)
return False
pay_amount = int("".join([i for i in tx.amount if i.isdigit()]))
if pay_amount < min_amount:
logger.log(
f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}"
)
return False
# Check if the payment was already used on a
used = laconic.app_deployments(
{"deployer": payment_address, "payment": tx.hash}, all=True
)
if len(used):
logger.log(f"{record.id}: payment {tx.hash} already used on deployment {used}")
return False
used = laconic.app_deployment_removals(
{"deployer": payment_address, "payment": tx.hash}, all=True
)
if len(used):
logger.log(
f"{record.id}: payment {tx.hash} already used on deployment removal {used}"
)
return False
return True
class LaconicRegistryClient:
def __init__(self, config_file, log_file=None):
self.config_file = config_file
@ -90,10 +167,94 @@ class LaconicRegistryClient:
self.cache = AttrDict(
{
"name_or_id": {},
"accounts": {},
"txs": {},
}
)
def list_records(self, criteria={}, all=False):
def whoami(self, refresh=False):
if not refresh and "whoami" in self.cache:
return self.cache["whoami"]
args = ["laconic", "-c", self.config_file, "registry", "account", "get"]
results = [
AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
]
if len(results):
self.cache["whoami"] = results[0]
return results[0]
return None
def get_owner(self, record, require=False):
bond = self.get_bond(record.bondId, require)
if bond:
return bond.owner
return bond
def get_account(self, address, refresh=False, require=False):
if not refresh and address in self.cache["accounts"]:
return self.cache["accounts"][address]
args = [
"laconic",
"-c",
self.config_file,
"registry",
"account",
"get",
"--address",
address,
]
results = [
AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
]
if len(results):
self.cache["accounts"][address] = results[0]
return results[0]
if require:
raise Exception("Cannot locate account:", address)
return None
def get_bond(self, id, require=False):
if id in self.cache.name_or_id:
return self.cache.name_or_id[id]
args = [
"laconic",
"-c",
self.config_file,
"registry",
"bond",
"get",
"--id",
id,
]
results = [
AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
]
self._add_to_cache(results)
if len(results):
return results[0]
if require:
raise Exception("Cannot locate bond:", id)
return None
def list_bonds(self):
args = ["laconic", "-c", self.config_file, "registry", "bond", "list"]
results = [
AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
]
self._add_to_cache(results)
return results
def list_records(self, criteria=None, all=False):
if criteria is None:
criteria = {}
args = ["laconic", "-c", self.config_file, "registry", "record", "list"]
if all:
@ -104,22 +265,17 @@ class LaconicRegistryClient:
args.append("--%s" % k)
args.append(str(v))
results = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args))]
results = [
AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
]
# Most recent records first
results.sort(key=lambda r: r.createTime)
results.reverse()
self._add_to_cache(results)
return results
def is_lrn(self, name_or_id: str):
if name_or_id:
return str(name_or_id).startswith("lrn://")
return False
def is_id(self, name_or_id: str):
return not self.is_lrn(name_or_id)
def _add_to_cache(self, records):
if not records:
return
@ -129,9 +285,10 @@ class LaconicRegistryClient:
if p.names:
for lrn in p.names:
self.cache["name_or_id"][lrn] = p
if p.attributes.type not in self.cache:
self.cache[p.attributes.type] = []
self.cache[p.attributes.type].append(p)
if p.attributes and p.attributes.type:
if p.attributes.type not in self.cache:
self.cache[p.attributes.type] = []
self.cache[p.attributes.type].append(p)
def resolve(self, name):
if not name:
@ -142,7 +299,9 @@ class LaconicRegistryClient:
args = ["laconic", "-c", self.config_file, "registry", "name", "resolve", name]
parsed = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args))]
parsed = [
AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
]
if parsed:
self._add_to_cache(parsed)
return parsed[0]
@ -158,7 +317,7 @@ class LaconicRegistryClient:
if name_or_id in self.cache.name_or_id:
return self.cache.name_or_id[name_or_id]
if self.is_lrn(name_or_id):
if is_lrn(name_or_id):
return self.resolve(name_or_id)
args = [
@ -172,7 +331,9 @@ class LaconicRegistryClient:
name_or_id,
]
parsed = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r]
parsed = [
AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
]
if len(parsed):
self._add_to_cache(parsed)
return parsed[0]
@ -181,38 +342,85 @@ class LaconicRegistryClient:
raise Exception("Cannot locate record:", name_or_id)
return None
def app_deployment_requests(self, all=True):
return self.list_records({"type": "ApplicationDeploymentRequest"}, all)
def get_tx(self, txHash, require=False):
if txHash in self.cache["txs"]:
return self.cache["txs"][txHash]
def app_deployments(self, all=True):
return self.list_records({"type": "ApplicationDeploymentRecord"}, all)
args = [
"laconic",
"-c",
self.config_file,
"registry",
"tokens",
"gettx",
"--hash",
txHash,
]
def app_deployment_removal_requests(self, all=True):
return self.list_records({"type": "ApplicationDeploymentRemovalRequest"}, all)
parsed = None
try:
parsed = AttrDict(json.loads(logged_cmd(self.log_file, *args)))
except: # noqa: E722
pass
def app_deployment_removals(self, all=True):
return self.list_records({"type": "ApplicationDeploymentRemovalRecord"}, all)
if parsed:
self.cache["txs"][txHash] = parsed
return parsed
def publish(self, record, names=[]):
if require:
raise Exception("Cannot locate tx:", hash)
def app_deployment_requests(self, criteria=None, all=True):
if criteria is None:
criteria = {}
criteria = criteria.copy()
criteria["type"] = "ApplicationDeploymentRequest"
return self.list_records(criteria, all)
def app_deployments(self, criteria=None, all=True):
if criteria is None:
criteria = {}
criteria = criteria.copy()
criteria["type"] = "ApplicationDeploymentRecord"
return self.list_records(criteria, all)
def app_deployment_removal_requests(self, criteria=None, all=True):
if criteria is None:
criteria = {}
criteria = criteria.copy()
criteria["type"] = "ApplicationDeploymentRemovalRequest"
return self.list_records(criteria, all)
def app_deployment_removals(self, criteria=None, all=True):
if criteria is None:
criteria = {}
criteria = criteria.copy()
criteria["type"] = "ApplicationDeploymentRemovalRecord"
return self.list_records(criteria, all)
def publish(self, record, names=None):
if names is None:
names = []
tmpdir = tempfile.mkdtemp()
try:
record_fname = os.path.join(tmpdir, "record.yml")
record_file = open(record_fname, 'w')
record_file = open(record_fname, "w")
yaml.dump(record, record_file)
record_file.close()
print(open(record_fname, 'r').read(), file=self.log_file)
print(open(record_fname, "r").read(), file=self.log_file)
new_record_id = json.loads(
logged_cmd(
self.log_file,
"laconic", "-c",
"laconic",
"-c",
self.config_file,
"registry",
"record",
"publish",
"--filename",
record_fname
)
record_fname,
)
)["id"]
for name in names:
self.set_name(name, new_record_id)
@ -221,10 +429,47 @@ class LaconicRegistryClient:
logged_cmd(self.log_file, "rm", "-rf", tmpdir)
def set_name(self, name, record_id):
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "registry", "name", "set", name, record_id)
logged_cmd(
self.log_file,
"laconic",
"-c",
self.config_file,
"registry",
"name",
"set",
name,
record_id,
)
def delete_name(self, name):
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "registry", "name", "delete", name)
logged_cmd(
self.log_file,
"laconic",
"-c",
self.config_file,
"registry",
"name",
"delete",
name,
)
def send_tokens(self, address, amount, type="alnt"):
args = [
"laconic",
"-c",
self.config_file,
"registry",
"tokens",
"send",
"--address",
address,
"--quantity",
str(amount),
"--type",
type,
]
return AttrDict(json.loads(logged_cmd(self.log_file, *args)))
def file_hash(filename):
@ -248,7 +493,9 @@ def determine_base_container(clone_dir, app_type="webapp"):
return base_container
def build_container_image(app_record, tag, extra_build_args=[], logger=None):
def build_container_image(app_record, tag, extra_build_args=None, logger=None):
if extra_build_args is None:
extra_build_args = []
tmpdir = tempfile.mkdtemp()
# TODO: determine if this code could be calling into the Python git library like setup-repositories
@ -265,9 +512,15 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
if github_token:
logger.log("Github token detected, setting it in the git environment")
git_config_args = [
"git", "config", "--global", f"url.https://{github_token}:@github.com/.insteadOf", "https://github.com/"
]
result = subprocess.run(git_config_args, stdout=logger.file, stderr=logger.file)
"git",
"config",
"--global",
f"url.https://{github_token}:@github.com/.insteadOf",
"https://github.com/",
]
result = subprocess.run(
git_config_args, stdout=logger.file, stderr=logger.file
)
result.check_returncode()
if ref:
# TODO: Determing branch or hash, and use depth 1 if we can.
@ -275,30 +528,50 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
# Never prompt
git_env["GIT_TERMINAL_PROMPT"] = "0"
try:
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=logger.file, stderr=logger.file)
subprocess.check_call(
["git", "clone", repo, clone_dir],
env=git_env,
stdout=logger.file,
stderr=logger.file,
)
except Exception as e:
logger.log(f"git clone failed. Is the repository {repo} private?")
raise e
try:
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=logger.file, stderr=logger.file)
subprocess.check_call(
["git", "checkout", ref],
cwd=clone_dir,
env=git_env,
stdout=logger.file,
stderr=logger.file,
)
except Exception as e:
logger.log(f"git checkout failed. Does ref {ref} exist?")
raise e
else:
# TODO: why is this code different vs the branch above (run vs check_call, and no prompt disable)?
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=logger.file, stderr=logger.file)
result = subprocess.run(
["git", "clone", "--depth", "1", repo, clone_dir],
stdout=logger.file,
stderr=logger.file,
)
result.check_returncode()
base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
base_container = determine_base_container(
clone_dir, app_record.attributes.app_type
)
logger.log("Building webapp ...")
build_command = [
sys.argv[0],
"--verbose",
"build-webapp",
"--source-repo", clone_dir,
"--tag", tag,
"--base-container", base_container
"--source-repo",
clone_dir,
"--tag",
tag,
"--base-container",
base_container,
]
if extra_build_args:
build_command.append("--extra-build-args")
@ -312,8 +585,11 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
def push_container_image(deployment_dir, logger):
logger.log("Pushing images ...")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"],
stdout=logger.file, stderr=logger.file)
result = subprocess.run(
[sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"],
stdout=logger.file,
stderr=logger.file,
)
result.check_returncode()
logger.log("Finished pushing images.")
@ -331,27 +607,35 @@ def deploy_to_k8s(deploy_record, deployment_dir, recreate, logger):
for command in commands_to_run:
logger.log(f"Running {command} command on deployment dir: {deployment_dir}")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
stdout=logger.file, stderr=logger.file)
result = subprocess.run(
[sys.argv[0], "deployment", "--dir", deployment_dir, command],
stdout=logger.file,
stderr=logger.file,
)
result.check_returncode()
logger.log(f"Finished {command} command on deployment dir: {deployment_dir}")
logger.log("Finished deploying to k8s.")
def publish_deployment(laconic: LaconicRegistryClient,
app_record,
deploy_record,
deployment_lrn,
dns_record,
dns_lrn,
deployment_dir,
app_deployment_request=None,
logger=None):
def publish_deployment(
laconic: LaconicRegistryClient,
app_record,
deploy_record,
deployment_lrn,
dns_record,
dns_lrn,
deployment_dir,
app_deployment_request=None,
webapp_deployer_record=None,
logger=None,
):
if not deploy_record:
deploy_ver = "0.0.1"
else:
deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1)
deploy_ver = "0.0.%d" % (
int(deploy_record.attributes.version.split(".")[-1]) + 1
)
if not dns_record:
dns_ver = "0.0.1"
@ -369,9 +653,7 @@ def publish_deployment(laconic: LaconicRegistryClient,
"version": dns_ver,
"name": fqdn,
"resource_type": "A",
"meta": {
"so": uniq.hex
},
"meta": {"so": uniq.hex},
}
}
if app_deployment_request:
@ -391,12 +673,19 @@ def publish_deployment(laconic: LaconicRegistryClient,
"dns": dns_id,
"meta": {
"config": file_hash(os.path.join(deployment_dir, "config.env")),
"so": uniq.hex
"so": uniq.hex,
},
}
}
if app_deployment_request:
new_deployment_record["record"]["request"] = app_deployment_request.id
if app_deployment_request.attributes.payment:
new_deployment_record["record"][
"payment"
] = app_deployment_request.attributes.payment
if webapp_deployer_record:
new_deployment_record["record"]["deployer"] = webapp_deployer_record.names[0]
if logger:
logger.log("Publishing ApplicationDeploymentRecord.")
@ -407,7 +696,9 @@ def publish_deployment(laconic: LaconicRegistryClient,
def hostname_for_deployment_request(app_deployment_request, laconic):
dns_name = app_deployment_request.attributes.dns
if not dns_name:
app = laconic.get_record(app_deployment_request.attributes.application, require=True)
app = laconic.get_record(
app_deployment_request.attributes.application, require=True
)
dns_name = generate_hostname_for_app(app)
elif dns_name.startswith("lrn://"):
record = laconic.get_record(dns_name, require=True)

View File

@ -24,7 +24,9 @@ from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy.webapp import (run_webapp,
deploy_webapp,
deploy_webapp_from_registry,
undeploy_webapp_from_registry)
undeploy_webapp_from_registry,
publish_webapp_deployer,
request_webapp_deployment)
from stack_orchestrator.deploy import deploy
from stack_orchestrator import version
from stack_orchestrator.deploy import deployment
@ -61,6 +63,8 @@ cli.add_command(run_webapp.command, "run-webapp")
cli.add_command(deploy_webapp.command, "deploy-webapp")
cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry")
cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry")
cli.add_command(publish_webapp_deployer.command, "publish-deployer-to-registry")
cli.add_command(request_webapp_deployment.command, "request-webapp-deployment")
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
cli.add_command(deploy.command, "deploy-system")
cli.add_command(deployment.command, "deployment")