deep-stack/stack-orchestrator
3
0
Fork 0
forked from cerc-io/stack-orchestrator
Code Pull Requests Activity

Add --dry-run option (#686)

Browse Source
This commit is contained in:
Thomas E Lackey 2023-12-13 22:56:40 -06:00 committed by GitHub
parent 88f66a3626
commit 1f4eb57069
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 325 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py
View File

@ -164,10 +164,11 @@ def dump_known_requests(filename, requests):
@click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world") @click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world")
@click.option("--record-namespace-dns", help="eg, crn://laconic/dns") @click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments") @click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@click.pass_context @click.pass_context
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,
request_id, discover, state_file, only_update_state, request_id, discover, state_file, only_update_state,
dns_suffix, record_namespace_dns, record_namespace_deployments): dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run):
if request_id and discover: if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr) print("Cannot specify both --request-id and --discover", file=sys.stderr)
sys.exit(2) sys.exit(2)
@ -196,7 +197,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
requests = laconic.app_deployment_requests() requests = laconic.app_deployment_requests()
if only_update_state: if only_update_state:
dump_known_requests(state_file, requests) if not dry_run:
dump_known_requests(state_file, requests)
return return
previous_requests = load_known_requests(state_file) previous_requests = load_known_requests(state_file)
@ -250,18 +252,19 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
for r in requests_to_execute: if not dry_run:
try: for r in requests_to_execute:
process_app_deployment_request( try:
ctx, process_app_deployment_request(
laconic, ctx,
r, laconic,
record_namespace_deployments, r,
record_namespace_dns, record_namespace_deployments,
dns_suffix, record_namespace_dns,
deployment_parent_dir, dns_suffix,
kube_config, deployment_parent_dir,
image_registry kube_config,
) image_registry
finally: )
dump_known_requests(state_file, [r]) finally:
dump_known_requests(state_file, [r])

608
stack_orchestrator/deploy/webapp/util.py
View File

@ -1,303 +1,305 @@
# Copyright © 2023 Vulcanize # Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by # it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# This program is distributed in the hope that it will be useful, # This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details. # GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>. # along with this program. If not, see <http:#www.gnu.org/licenses/>.
import hashlib import hashlib
import json import json
import os import os
import random import random
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import uuid import uuid
import yaml import yaml
class AttrDict(dict): class AttrDict(dict):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(AttrDict, self).__init__(*args, **kwargs) super(AttrDict, self).__init__(*args, **kwargs)
self.__dict__ = self self.__dict__ = self
def __getattribute__(self, attr): def __getattribute__(self, attr):
__dict__ = super(AttrDict, self).__getattribute__("__dict__") __dict__ = super(AttrDict, self).__getattribute__("__dict__")
if attr in __dict__: if attr in __dict__:
v = super(AttrDict, self).__getattribute__(attr) v = super(AttrDict, self).__getattribute__(attr)
if isinstance(v, dict): if isinstance(v, dict):
return AttrDict(v) return AttrDict(v)
return v return v
def cmd(*vargs): def cmd(*vargs):
try: try:
result = subprocess.run(vargs, capture_output=True) result = subprocess.run(vargs, capture_output=True)
result.check_returncode() result.check_returncode()
return result.stdout.decode() return result.stdout.decode()
except Exception as err: except Exception as err:
print(result.stderr.decode()) print(result.stderr.decode())
raise err raise err
class LaconicRegistryClient: class LaconicRegistryClient:
def __init__(self, config_file): def __init__(self, config_file):
self.config_file = config_file self.config_file = config_file
self.cache = AttrDict( self.cache = AttrDict(
{ {
"name_or_id": {}, "name_or_id": {},
} }
) )
def list_records(self, criteria={}, all=False): def list_records(self, criteria={}, all=False):
args = ["laconic", "-c", self.config_file, "cns", "record", "list"] args = ["laconic", "-c", self.config_file, "cns", "record", "list"]
if all: if all:
args.append("--all") args.append("--all")
if criteria: if criteria:
for k, v in criteria.items(): for k, v in criteria.items():
args.append("--%s" % k) args.append("--%s" % k)
args.append(str(v)) args.append(str(v))
results = [AttrDict(r) for r in json.loads(cmd(*args))] results = [AttrDict(r) for r in json.loads(cmd(*args))]
# Most recent records first # Most recent records first
results.sort(key=lambda r: r.createTime) results.sort(key=lambda r: r.createTime)
results.reverse() results.reverse()
return results return results
def is_crn(self, name_or_id: str): def is_crn(self, name_or_id: str):
if name_or_id: if name_or_id:
return str(name_or_id).startswith("crn://") return str(name_or_id).startswith("crn://")
return False return False
def is_id(self, name_or_id: str): def is_id(self, name_or_id: str):
return not self.is_crn(name_or_id) return not self.is_crn(name_or_id)
def _add_to_cache(self, records): def _add_to_cache(self, records):
if not records: if not records:
return return
for p in records: for p in records:
self.cache["name_or_id"][p.id] = p self.cache["name_or_id"][p.id] = p
if p.names: if p.names:
for crn in p.names: for crn in p.names:
self.cache["name_or_id"][crn] = p self.cache["name_or_id"][crn] = p
if p.attributes.type not in self.cache: if p.attributes.type not in self.cache:
self.cache[p.attributes.type] = [] self.cache[p.attributes.type] = []
self.cache[p.attributes.type].append(p) self.cache[p.attributes.type].append(p)
def resolve(self, name): def resolve(self, name):
if not name: if not name:
return None return None
if name in self.cache.name_or_id: if name in self.cache.name_or_id:
return self.cache.name_or_id[name] return self.cache.name_or_id[name]
args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name] args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name]
parsed = [AttrDict(r) for r in json.loads(cmd(*args))] parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
if parsed: if parsed:
self._add_to_cache(parsed) self._add_to_cache(parsed)
return parsed[0] return parsed[0]
return None return None
def get_record(self, name_or_id, require=False): def get_record(self, name_or_id, require=False):
if not name_or_id: if not name_or_id:
if require: if require:
raise Exception("Cannot locate record:", name_or_id) raise Exception("Cannot locate record:", name_or_id)
return None return None
if name_or_id in self.cache.name_or_id: if name_or_id in self.cache.name_or_id:
return self.cache.name_or_id[name_or_id] return self.cache.name_or_id[name_or_id]
if self.is_crn(name_or_id): if self.is_crn(name_or_id):
return self.resolve(name_or_id) return self.resolve(name_or_id)
args = [ args = [
"laconic", "laconic",
"-c", "-c",
self.config_file, self.config_file,
"cns", "cns",
"record", "record",
"get", "get",
"--id", "--id",
name_or_id, name_or_id,
] ]
parsed = [AttrDict(r) for r in json.loads(cmd(*args))] parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
if len(parsed): if len(parsed):
self._add_to_cache(parsed) self._add_to_cache(parsed)
return parsed[0] return parsed[0]
if require: if require:
raise Exception("Cannot locate record:", name_or_id) raise Exception("Cannot locate record:", name_or_id)
return None return None
def app_deployment_requests(self): def app_deployment_requests(self):
return self.list_records({"type": "ApplicationDeploymentRequest"}, True) return self.list_records({"type": "ApplicationDeploymentRequest"}, True)
def app_deployments(self): def app_deployments(self):
return self.list_records({"type": "ApplicationDeploymentRecord"}) return self.list_records({"type": "ApplicationDeploymentRecord"})
def publish(self, record, names=[]): def publish(self, record, names=[]):
tmpdir = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp()
try: try:
record_fname = os.path.join(tmpdir, "record.yml") record_fname = os.path.join(tmpdir, "record.yml")
record_file = open(record_fname, 'w') record_file = open(record_fname, 'w')
yaml.dump(record, record_file) yaml.dump(record, record_file)
record_file.close() record_file.close()
print(open(record_fname, 'r').read()) print(open(record_fname, 'r').read())
new_record_id = json.loads( new_record_id = json.loads(
cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
)["id"] )["id"]
for name in names: for name in names:
cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id) cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id)
return new_record_id return new_record_id
finally: finally:
cmd("rm", "-rf", tmpdir) cmd("rm", "-rf", tmpdir)
def file_hash(filename): def file_hash(filename):
return hashlib.sha1(open(filename).read().encode()).hexdigest() return hashlib.sha1(open(filename).read().encode()).hexdigest()
def build_container_image(app_record, tag, extra_build_args=[]): def build_container_image(app_record, tag, extra_build_args=[]):
tmpdir = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp()
try: try:
record_id = app_record["id"] record_id = app_record["id"]
ref = app_record.attributes.repository_ref ref = app_record.attributes.repository_ref
repo = random.choice(app_record.attributes.repository) repo = random.choice(app_record.attributes.repository)
clone_dir = os.path.join(tmpdir, record_id) clone_dir = os.path.join(tmpdir, record_id)
print(f"Cloning repository {repo} to {clone_dir} ...") print(f"Cloning repository {repo} to {clone_dir} ...")
if ref: if ref:
result = subprocess.run(["git", "clone", "--depth", "1", "--branch", ref, repo, clone_dir]) # TODO: Determing branch or hash, and use depth 1 if we can.
result.check_returncode() result = subprocess.run(["git", "clone", repo, clone_dir])
else: result.check_returncode()
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) subprocess.check_call(["git", "checkout", ref], cwd=clone_dir)
result.check_returncode() else:
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
print("Building webapp ...") result.check_returncode()
build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag]
if extra_build_args: print("Building webapp ...")
build_command.append("--extra-build-args") build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag]
build_command.append(" ".join(extra_build_args)) if extra_build_args:
build_command.append("--extra-build-args")
result = subprocess.run(build_command) build_command.append(" ".join(extra_build_args))
result.check_returncode()
finally: result = subprocess.run(build_command)
cmd("rm", "-rf", tmpdir) result.check_returncode()
finally:
cmd("rm", "-rf", tmpdir)
def push_container_image(deployment_dir):
print("Pushing image ...")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) def push_container_image(deployment_dir):
result.check_returncode() print("Pushing image ...")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"])
result.check_returncode()
def deploy_to_k8s(deploy_record, deployment_dir):
if not deploy_record:
command = "up" def deploy_to_k8s(deploy_record, deployment_dir):
else: if not deploy_record:
command = "update" command = "up"
else:
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) command = "update"
result.check_returncode()
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command])
result.check_returncode()
def publish_deployment(laconic: LaconicRegistryClient,
app_record,
deploy_record, def publish_deployment(laconic: LaconicRegistryClient,
deployment_crn, app_record,
dns_record, deploy_record,
dns_crn, deployment_crn,
deployment_dir, dns_record,
app_deployment_request=None): dns_crn,
if not deploy_record: deployment_dir,
deploy_ver = "0.0.1" app_deployment_request=None):
else: if not deploy_record:
deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1) deploy_ver = "0.0.1"
else:
if not dns_record: deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1)
dns_ver = "0.0.1"
else: if not dns_record:
dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1) dns_ver = "0.0.1"
else:
spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml"))) dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1)
fqdn = spec["network"]["http-proxy"][0]["host-name"]
spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml")))
uniq = uuid.uuid4() fqdn = spec["network"]["http-proxy"][0]["host-name"]
new_dns_record = { uniq = uuid.uuid4()
"record": {
"type": "DnsRecord", new_dns_record = {
"version": dns_ver, "record": {
"name": fqdn, "type": "DnsRecord",
"resource_type": "A", "version": dns_ver,
"meta": { "name": fqdn,
"so": uniq.hex "resource_type": "A",
}, "meta": {
} "so": uniq.hex
} },
if app_deployment_request: }
new_dns_record["record"]["request"] = app_deployment_request.id }
if app_deployment_request:
dns_id = laconic.publish(new_dns_record, [dns_crn]) new_dns_record["record"]["request"] = app_deployment_request.id
new_deployment_record = { dns_id = laconic.publish(new_dns_record, [dns_crn])
"record": {
"type": "ApplicationDeploymentRecord", new_deployment_record = {
"version": deploy_ver, "record": {
"url": f"https://{fqdn}", "type": "ApplicationDeploymentRecord",
"name": app_record.attributes.name, "version": deploy_ver,
"application": app_record.id, "url": f"https://{fqdn}",
"dns": dns_id, "name": app_record.attributes.name,
"meta": { "application": app_record.id,
"config": file_hash(os.path.join(deployment_dir, "config.env")), "dns": dns_id,
"so": uniq.hex "meta": {
}, "config": file_hash(os.path.join(deployment_dir, "config.env")),
} "so": uniq.hex
} },
if app_deployment_request: }
new_deployment_record["record"]["request"] = app_deployment_request.id }
if app_deployment_request:
deployment_id = laconic.publish(new_deployment_record, [deployment_crn]) new_deployment_record["record"]["request"] = app_deployment_request.id
return {"dns": dns_id, "deployment": deployment_id}
deployment_id = laconic.publish(new_deployment_record, [deployment_crn])
return {"dns": dns_id, "deployment": deployment_id}
def hostname_for_deployment_request(app_deployment_request, laconic):
dns_name = app_deployment_request.attributes.dns
if not dns_name: def hostname_for_deployment_request(app_deployment_request, laconic):
app = laconic.get_record(app_deployment_request.attributes.application, require=True) dns_name = app_deployment_request.attributes.dns
dns_name = generate_hostname_for_app(app) if not dns_name:
elif dns_name.startswith("crn://"): app = laconic.get_record(app_deployment_request.attributes.application, require=True)
record = laconic.get_record(dns_name, require=True) dns_name = generate_hostname_for_app(app)
dns_name = record.attributes.name elif dns_name.startswith("crn://"):
return dns_name record = laconic.get_record(dns_name, require=True)
dns_name = record.attributes.name
return dns_name
def generate_hostname_for_app(app):
last_part = app.attributes.name.split("/")[-1]
m = hashlib.sha256() def generate_hostname_for_app(app):
m.update(app.attributes.name.encode()) last_part = app.attributes.name.split("/")[-1]
m.update(b"|") m = hashlib.sha256()
if isinstance(app.attributes.repository, list): m.update(app.attributes.name.encode())
m.update(app.attributes.repository[0].encode()) m.update(b"|")
else: if isinstance(app.attributes.repository, list):
m.update(app.attributes.repository.encode()) m.update(app.attributes.repository[0].encode())
return "%s-%s" % (last_part, m.hexdigest()[0:10]) else:
m.update(app.attributes.repository.encode())
return "%s-%s" % (last_part, m.hexdigest()[0:10])