Compare commits

..
Author SHA1 Message Date
zramsay 4cbc84b388 update url 2024-09-13 09:58:51 -04:00
25 changed files with 327 additions and 1322 deletions
@@ -39,7 +39,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Build local shiv package"
id: build
run: |
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
@@ -35,7 +35,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
+1 -1
View File
@@ -32,7 +32,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv==1.0.6
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
@@ -3,7 +3,7 @@ name: act-runner
description: "Local act-runner"
repos:
- git.vdb.to/cerc-io/hosting
- gitea.com/telackey/act_runner@telackey/entrypoint
- git.vdb.to/cerc-io/act_runner
containers:
- cerc/act-runner
- cerc/act-runner-task-executor
@@ -27,25 +27,3 @@ The Package Registry Stack supports a build environment that requires a package
```
* The local gitea registry can now be accessed at <http://localhost:3000> (the username and password can be taken from the deployment logs)
* Configure the hostname `gitea.local`:
Update `/etc/hosts`:
```bash
sudo nano /etc/hosts
# Add the following line
127.0.0.1 gitea.local
```
Check resolution:
```bash
ping gitea.local
PING gitea.local (127.0.0.1) 56(84) bytes of data.
64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.147 ms
64 bytes from localhost (127.0.0.1): icmp_seq=2 ttl=64 time=0.033 ms
...
```
+26 -31
View File
@@ -114,27 +114,22 @@ class ClusterInfo:
nodeports.append(service)
return nodeports
def get_ingress(self, use_tls=False, certificate_by_host={}, cluster_issuer="letsencrypt-prod"):
def get_ingress(self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"):
# No ingress for a deployment that has no http-proxy defined, for now
http_proxy_info_list = self.spec.get_http_proxy()
if not http_proxy_info_list:
return None
tls = [] if use_tls else None
rules = []
for http_proxy_info in http_proxy_info_list:
ingress = None
if http_proxy_info_list:
# TODO: handle multiple definitions
http_proxy_info = http_proxy_info_list[0]
if opts.o.debug:
print(f"http-proxy: {http_proxy_info}")
# TODO: good enough parsing for webapp deployment for now
host_name = http_proxy_info["host-name"]
certificate = certificate_by_host[host_name] if host_name in certificate_by_host else None
if use_tls:
tls.append(client.V1IngressTLS(
hosts=certificate["spec"]["dnsNames"] if certificate else [host_name],
secret_name=certificate["spec"]["secretName"] if certificate else f"{self.app_name}-{host_name}-tls"
))
rules = []
tls = [client.V1IngressTLS(
hosts=certificate["spec"]["dnsNames"] if certificate else [host_name],
secret_name=certificate["spec"]["secretName"] if certificate else f"{self.app_name}-tls"
)] if use_tls else None
paths = []
for route in http_proxy_info["routes"]:
path = route["path"]
@@ -161,24 +156,24 @@ class ClusterInfo:
paths=paths
)
))
spec = client.V1IngressSpec(
tls=tls,
rules=rules
)
spec = client.V1IngressSpec(
tls=tls,
rules=rules
)
ingress_annotations = {
"kubernetes.io/ingress.class": "nginx",
}
if not certificate:
ingress_annotations["cert-manager.io/cluster-issuer"] = cluster_issuer
ingress_annotations = {
"kubernetes.io/ingress.class": "nginx",
}
if not certificate:
ingress_annotations["cert-manager.io/cluster-issuer"] = cluster_issuer
ingress = client.V1Ingress(
metadata=client.V1ObjectMeta(
name=f"{self.app_name}-ingress",
annotations=ingress_annotations
),
spec=spec
)
ingress = client.V1Ingress(
metadata=client.V1ObjectMeta(
name=f"{self.app_name}-ingress",
annotations=ingress_annotations
),
spec=spec
)
return ingress
# TODO: suppoprt multiple services
+27 -40
View File
@@ -227,18 +227,15 @@ class K8sDeployer(Deployer):
self._create_volume_data()
self._create_deployment()
http_proxy_info_list = self.cluster_info.spec.get_http_proxy()
http_proxy_info = self.cluster_info.spec.get_http_proxy()
# Note: at present we don't support tls for kind (and enabling tls causes errors)
use_tls = http_proxy_info_list and not self.is_kind()
certificate_by_host = {}
if use_tls:
for http_proxy_info in http_proxy_info_list:
certificate = self._find_certificate_for_host_name(http_proxy_info["host-name"])
if opts.o.debug and certificate:
print(f"Using existing certificate: {certificate}")
certificate_by_host[http_proxy_info["host-name"]] = certificate
use_tls = http_proxy_info and not self.is_kind()
certificate = self._find_certificate_for_host_name(http_proxy_info[0]["host-name"]) if use_tls else None
if opts.o.debug:
if certificate:
print(f"Using existing certificate: {certificate}")
ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=use_tls, certificate_by_host=certificate_by_host)
ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=use_tls, certificate=certificate)
if ingress:
if opts.o.debug:
print(f"Sending this ingress: {ingress}")
@@ -384,46 +381,36 @@ class K8sDeployer(Deployer):
if not pods:
return
tls_by_host = {}
hostname = "?"
ip = "?"
tls = "?"
try:
ingress = self.networking_api.read_namespaced_ingress(namespace=self.k8s_namespace,
name=self.cluster_info.get_ingress().metadata.name)
cert = self.custom_obj_api.get_namespaced_custom_object(
group="cert-manager.io",
version="v1",
namespace=self.k8s_namespace,
plural="certificates",
name=ingress.spec.tls[0].secret_name
)
hostname = ingress.spec.rules[0].host
ip = ingress.status.load_balancer.ingress[0].ip
for rule in ingress.spec.rules:
hostname = rule.host
tls_spec = next((tls for tls in ingress.spec.tls if hostname in tls.hosts), None)
if tls_spec:
cert = self.custom_obj_api.get_namespaced_custom_object(
group="cert-manager.io",
version="v1",
namespace=self.k8s_namespace,
plural="certificates",
name=tls_spec.secret_name
)
tls = "notBefore: %s; notAfter: %s; names: %s" % (
cert["status"]["notBefore"], cert["status"]["notAfter"], tls_spec.hosts
)
tls_by_host[hostname] = tls
else:
tls_by_host[hostname] = None
tls = "notBefore: %s; notAfter: %s; names: %s" % (
cert["status"]["notBefore"], cert["status"]["notAfter"], ingress.spec.tls[0].hosts
)
except: # noqa: E722
pass
print("Ingress:")
if len(tls_by_host) == 0:
print("\tHostname:", "?")
print("\tIP:", "?")
print("\tTLS:", "?")
print("")
for hostname, tls in tls_by_host.items():
print("\tHostname:", hostname)
print("\tIP:", ip)
print("\tTLS:", tls)
print("")
print("\tHostname:", hostname)
print("\tIP:", ip)
print("\tTLS:", tls)
print("")
print("Pods:")
for p in pods:
if p.metadata.deletion_timestamp:
print(f"\t{p.metadata.namespace}/{p.metadata.name}: Terminating ({p.metadata.deletion_timestamp})")
@@ -13,16 +13,12 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import shutil
from typing import List
import click
import os
import yaml
from pathlib import Path
from urllib.parse import urlparse
from tempfile import NamedTemporaryFile
from stack_orchestrator import constants
from stack_orchestrator.util import error_exit, global_options2
from stack_orchestrator.deploy.deployment_create import init_operation, create_operation
from stack_orchestrator.deploy.deploy import create_deploy_context
@@ -40,38 +36,25 @@ def _fixup_container_tag(deployment_dir: str, image: str):
wfile.write(contents)
def _fixup_url_spec(spec_file_name: str, urls: List[str]):
def _fixup_url_spec(spec_file_name: str, url: str):
# url is like: https://example.com/path
parsed_url = urlparse(url)
http_proxy_spec = f'''
http-proxy:
- host-name: {parsed_url.hostname}
routes:
- path: '{parsed_url.path if parsed_url.path else "/"}'
proxy-to: webapp:80
'''
spec_file_path = Path(spec_file_name)
# Load existing spec
with open(spec_file_path, "r") as file:
spec_data = yaml.safe_load(file) or {}
# Build new http-proxy entries
http_proxy_entries = []
for url in urls:
parsed_url = urlparse(url)
http_proxy_entries.append({
"host-name": parsed_url.hostname,
"routes": [
{
"path": parsed_url.path if parsed_url.path else "/",
"proxy-to": "webapp:80"
}
]
})
# Update the spec
if "network" not in spec_data:
spec_data["network"] = {}
spec_data["network"]["http-proxy"] = http_proxy_entries
# Write back the updated YAML
with open(spec_file_path, "w") as file:
yaml.dump(spec_data, file, default_flow_style=False, sort_keys=False)
with open(spec_file_path) as rfile:
contents = rfile.read()
contents = contents + http_proxy_spec
with open(spec_file_path, "w") as wfile:
wfile.write(contents)
def create_deployment(ctx, deployment_dir, image, urls, kube_config, image_registry, env_file):
def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
# Do the equivalent of:
# 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml
# --config (eqivalent of the contents of my-config.env)
@@ -103,7 +86,7 @@ def create_deployment(ctx, deployment_dir, image, urls, kube_config, image_regis
None
)
# Add the TLS and DNS spec
_fixup_url_spec(spec_file_name, urls)
_fixup_url_spec(spec_file_name, url)
create_operation(
deploy_command_context,
spec_file_name,
@@ -116,20 +99,6 @@ def create_deployment(ctx, deployment_dir, image, urls, kube_config, image_regis
os.remove(spec_file_name)
def update_deployment(deployment_dir, image, urls, env_file):
# Update config if required
if env_file:
deployment_config_file = os.path.join(deployment_dir, "config.env")
shutil.copyfile(env_file, deployment_config_file)
# Update existing deployment spec with new urls
_fixup_url_spec(os.path.join(deployment_dir, constants.spec_file_name), urls)
# Update the image name if required
if image:
_fixup_container_tag(deployment_dir, image)
@click.group()
@click.pass_context
def command(ctx):
@@ -151,4 +120,4 @@ def command(ctx):
def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
'''create a deployment for the specified webapp container'''
return create_deployment(ctx, deployment_dir, image, [url], kube_config, image_registry, env_file)
return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file)
@@ -20,7 +20,6 @@ import shutil
import sys
import tempfile
import time
from typing import List
import uuid
import yaml
@@ -34,29 +33,25 @@ from stack_orchestrator.deploy.webapp.util import (
LaconicRegistryClient,
TimedLogger,
build_container_image,
confirm_auction,
push_container_image,
file_hash,
deploy_to_k8s,
publish_deployment,
get_requested_names,
hostnames_for_deployment_request,
hostname_for_deployment_request,
generate_hostname_for_app,
match_owner,
skip_by_tag,
confirm_payment,
load_known_requests,
)
def process_app_deployment_request( # noqa
def process_app_deployment_request(
ctx,
laconic: LaconicRegistryClient,
app_deployment_request,
deployment_record_namespace,
dns_record_namespace,
default_dns_suffix,
dns_value,
deployment_parent_dir,
kube_config,
image_registry,
@@ -78,54 +73,45 @@ def process_app_deployment_request( # noqa
logger.log(f"Retrieved app record {app_deployment_request.attributes.application}")
# 2. determine dns
requested_names = hostnames_for_deployment_request(app_deployment_request, laconic)
logger.log(f"Determined requested name(s): {','.join(str(x) for x in requested_names)}")
requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
logger.log(f"Determined requested name: {requested_name}")
fqdns = []
for requested_name in requested_names:
if "." in requested_name:
if "allow" == fqdn_policy or "preexisting" == fqdn_policy:
fqdns.append(requested_name)
else:
raise Exception(
f"{requested_name} is invalid: only unqualified hostnames are allowed."
)
if "." in requested_name:
if "allow" == fqdn_policy or "preexisting" == fqdn_policy:
fqdn = requested_name
else:
fqdns.append(f"{requested_name}.{default_dns_suffix}")
raise Exception(
f"{requested_name} is invalid: only unqualified hostnames are allowed."
)
else:
fqdn = f"{requested_name}.{default_dns_suffix}"
# Normalize case (just in case)
fqdns = [fqdn.lower() for fqdn in fqdns]
fqdn = fqdn.lower()
# 3. check ownership of existing dnsrecord(s) vs this request
dns_lrns = []
existing_dns_records_by_lrns = {}
for fqdn in fqdns:
dns_lrn = f"{dns_record_namespace}/{fqdn}"
dns_lrns.append(dns_lrn)
dns_record = laconic.get_record(dns_lrn)
existing_dns_records_by_lrns[dns_lrn] = dns_record
if dns_record:
matched_owner = match_owner(app_deployment_request, dns_record)
if not matched_owner and dns_record.attributes.request:
matched_owner = match_owner(
app_deployment_request,
laconic.get_record(dns_record.attributes.request, require=True),
)
if matched_owner:
logger.log(f"Matched DnsRecord ownership for {fqdn}: {matched_owner}")
else:
raise Exception(
"Unable to confirm ownership of DnsRecord %s for request %s"
% (dns_lrn, app_deployment_request.id)
)
elif "preexisting" == fqdn_policy:
raise Exception(
f"No pre-existing DnsRecord {dns_lrn} could be found for request {app_deployment_request.id}."
# 3. check ownership of existing dnsrecord vs this request
dns_lrn = f"{dns_record_namespace}/{fqdn}"
dns_record = laconic.get_record(dns_lrn)
if dns_record:
matched_owner = match_owner(app_deployment_request, dns_record)
if not matched_owner and dns_record.attributes.request:
matched_owner = match_owner(
app_deployment_request,
laconic.get_record(dns_record.attributes.request, require=True),
)
if matched_owner:
logger.log(f"Matched DnsRecord ownership: {matched_owner}")
else:
raise Exception(
"Unable to confirm ownership of DnsRecord %s for request %s"
% (dns_lrn, app_deployment_request.id)
)
elif "preexisting" == fqdn_policy:
raise Exception(
f"No pre-existing DnsRecord {dns_lrn} could be found for request {app_deployment_request.id}."
)
# 4. get build and runtime config from request
env = {}
if app_deployment_request.attributes.config:
@@ -156,49 +142,30 @@ def process_app_deployment_request( # noqa
# 5. determine new or existing deployment
# a. check for deployment lrn
app_deployment_lrns = [f"{deployment_record_namespace}/{fqdn}" for fqdn in fqdns]
app_deployment_lrn = f"{deployment_record_namespace}/{fqdn}"
if app_deployment_request.attributes.deployment:
app_deployment_lrns = [app_deployment_request.attributes.deployment]
if not app_deployment_lrns[0].startswith(deployment_record_namespace):
raise Exception(
"Deployment LRN %s is not in a supported namespace"
% app_deployment_request.attributes.deployment
)
app_deployment_lrn = app_deployment_request.attributes.deployment
if not app_deployment_lrn.startswith(deployment_record_namespace):
raise Exception(
"Deployment LRN %s is not in a supported namespace"
% app_deployment_request.attributes.deployment
)
# Target deployment dir
deployment_dir = os.path.join(deployment_parent_dir, fqdns[-1])
# Existing deployment record: take the first lrn that resolves
deployment_record = None
fqdns_to_release = []
existing_deployment_dir = deployment_dir # Default to target dir in case the app had been undeployed
for app_deployment_lrn in app_deployment_lrns:
deployment_record = laconic.get_record(app_deployment_lrn)
if deployment_record:
# Determine the deployment dir for existing deployment
dir_name = deployment_record.attributes.url.replace("https://", "")
existing_deployment_dir = os.path.join(deployment_parent_dir, dir_name)
if not os.path.exists(existing_deployment_dir):
raise Exception(
"Deployment record %s exists, but not deployment dir %s. Please remove name."
% (app_deployment_lrn, existing_deployment_dir)
)
previous_app_deployment_lrns: List[str] = deployment_record.names
previous_fqdns = [lrn.removeprefix(f"{deployment_record_namespace}/") for lrn in previous_app_deployment_lrns]
fqdns_to_release = list(set(previous_fqdns) - set(fqdns))
break
# Use the last fqdn for unique deployment container tag
deployment_record = laconic.get_record(app_deployment_lrn)
deployment_dir = os.path.join(deployment_parent_dir, fqdn)
# At present we use this to generate a unique but stable ID for the app's host container
# TODO: implement support to derive this transparently from the already-unique deployment id
unique_deployment_id = hashlib.md5(fqdns[-1].encode()).hexdigest()[:16]
unique_deployment_id = hashlib.md5(fqdn.encode()).hexdigest()[:16]
deployment_config_file = os.path.join(deployment_dir, "config.env")
deployment_container_tag = "laconic-webapp/%s:local" % unique_deployment_id
app_image_shared_tag = f"laconic-webapp/{app.id}:local"
# b. check for deployment directory (create if necessary)
if not os.path.exists(existing_deployment_dir):
if not os.path.exists(deployment_dir):
if deployment_record:
raise Exception(
"Deployment record %s exists, but not deployment dir %s. Please remove name."
% (app_deployment_lrn, deployment_dir)
)
logger.log(
f"Creating webapp deployment in: {deployment_dir} with container id: {deployment_container_tag}"
)
@@ -206,23 +173,13 @@ def process_app_deployment_request( # noqa
ctx,
deployment_dir,
deployment_container_tag,
[f"https://{fqdn}" for fqdn in fqdns],
f"https://{fqdn}",
kube_config,
image_registry,
env_filename,
)
else:
# Rename deployment dir according to new request (last fqdn from given dns)
os.rename(existing_deployment_dir, deployment_dir)
# Update the image name deployment_container_tag
# Skip for redeployment as deployment_container_tag won't get built
updated_image = None
if deployment_record.attributes.application != app.id:
updated_image = deployment_container_tag
# Update the existing deployment
deploy_webapp.update_deployment(deployment_dir, updated_image, [f"https://{fqdn}" for fqdn in fqdns], env_filename)
elif env_filename:
shutil.copyfile(env_filename, deployment_config_file)
needs_k8s_deploy = False
if force_rebuild:
@@ -272,11 +229,10 @@ def process_app_deployment_request( # noqa
else:
logger.log("Requested app is already deployed, skipping build and image push")
# 7. restart deployment on config or url spec change
# 7. update config (if needed)
if (
not deployment_record
or file_hash(os.path.join(deployment_dir, "config.env")) != deployment_record.attributes.meta.config
or len(fqdns_to_release) != 0
or file_hash(deployment_config_file) != deployment_record.attributes.meta.config
):
needs_k8s_deploy = True
@@ -289,31 +245,24 @@ def process_app_deployment_request( # noqa
laconic,
app,
deployment_record,
app_deployment_lrns,
existing_dns_records_by_lrns,
dns_lrns,
dns_record_namespace,
app_deployment_lrn,
dns_record,
dns_lrn,
deployment_dir,
dns_value,
app_deployment_request,
webapp_deployer_record,
logger,
)
logger.log("Publication complete.")
# 9. delete unused names from previous deployment (app and dns)
for fqdn in fqdns_to_release:
# Delete app deployment name and DNS name
deployment_name = "{deployment_record_namespace}/{fqdn}"
dns_name = "{deployment_record_namespace}/{fqdn}"
logger.log(f"Removing names {deployment_name} and {dns_name}")
laconic.delete_name(deployment_name)
laconic.delete_name(dns_name)
logger.log("END - process_app_deployment_request")
def load_known_requests(filename):
if filename and os.path.exists(filename):
return json.load(open(filename, "r"))
return {}
def dump_known_requests(filename, requests, status="SEEN"):
if not filename:
return
@@ -359,7 +308,6 @@ def dump_known_requests(filename, requests, status="SEEN"):
help="How to handle requests with an FQDN: prohibit, allow, preexisting",
default="prohibit",
)
@click.option("--ip", help="IP address of the k8s deployment (to be set in DNS record)", default=None)
@click.option("--record-namespace-dns", help="eg, lrn://laconic/dns", required=True)
@click.option(
"--record-namespace-deployments",
@@ -402,12 +350,6 @@ def dump_known_requests(filename, requests, status="SEEN"):
"my payment address are examined).",
is_flag=True,
)
@click.option(
"--auction-requests",
help="Handle requests with auction id set (skips payment confirmation).",
is_flag=True,
default=False,
)
@click.option(
"--config-upload-dir",
help="The directory containing uploaded config.",
@@ -416,9 +358,6 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option(
"--private-key-file", help="The private key for decrypting config.", required=True
)
@click.option(
"--registry-lock-file", help="File path to use for registry mutex lock", default=None
)
@click.option(
"--private-key-passphrase",
help="The passphrase for the private key.",
@@ -437,7 +376,6 @@ def command( # noqa: C901
only_update_state,
dns_suffix,
fqdn_policy,
ip,
record_namespace_dns,
record_namespace_deployments,
dry_run,
@@ -452,8 +390,6 @@ def command( # noqa: C901
private_key_file,
private_key_passphrase,
all_requests,
auction_requests,
registry_lock_file,
):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
@@ -486,13 +422,6 @@ def command( # noqa: C901
)
sys.exit(2)
if fqdn_policy == "allow" and not ip:
print(
"--ip is required with 'allow' fqdn-policy",
file=sys.stderr,
)
sys.exit(2)
tempdir = tempfile.mkdtemp()
gpg = gnupg.GPG(gnupghome=tempdir)
@@ -512,7 +441,7 @@ def command( # noqa: C901
include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file)
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr)
webapp_deployer_record = laconic.get_record(lrn, require=True)
payment_address = webapp_deployer_record.attributes.paymentAddress
main_logger.log(f"Payment address: {payment_address}")
@@ -570,26 +499,21 @@ def command( # noqa: C901
result = "ERROR"
continue
requested_names = get_requested_names(r)
if len(requested_names) == 0:
requested_names = [generate_hostname_for_app(app)]
requested_name = r.attributes.dns
if not requested_name:
requested_name = generate_hostname_for_app(app)
main_logger.log(
"Generating name %s for request %s." % (requested_names[0], r.id)
"Generating name %s for request %s." % (requested_name, r.id)
)
# Skip request if any of the names is superseded
for requested_name in requested_names:
if (
requested_name in skipped_by_name
or requested_name in requests_by_name
):
main_logger.log(
"Ignoring request %s, it has been superseded." % r.id
)
result = "SKIP"
break
if result == "SKIP":
if (
requested_name in skipped_by_name
or requested_name in requests_by_name
):
main_logger.log(
"Ignoring request %s, it has been superseded." % r.id
)
result = "SKIP"
continue
if skip_by_tag(r, include_tags, exclude_tags):
@@ -597,20 +521,15 @@ def command( # noqa: C901
"Skipping request %s, filtered by tag (include %s, exclude %s, present %s)"
% (r.id, include_tags, exclude_tags, r.attributes.tags)
)
for requested_name in requested_names:
skipped_by_name[requested_name] = r
skipped_by_name[requested_name] = r
result = "SKIP"
continue
main_logger.log(
"Found pending request %s to run application %s on %s."
% (r.id, r.attributes.application, ','.join(str(x) for x in requested_names))
% (r.id, r.attributes.application, requested_name)
)
# Set request for on of the names
requests_by_name[requested_names[-1]] = r
requests_by_name[requested_name] = r
except Exception as e:
result = "ERROR"
main_logger.log(f"ERROR examining request {r.id}: " + str(e))
@@ -663,29 +582,8 @@ def command( # noqa: C901
requests_to_check_for_payment.append(r)
requests_to_execute = []
for r in requests_to_check_for_payment:
if r.attributes.auction:
if auction_requests:
if confirm_auction(
laconic,
r,
lrn,
payment_address,
main_logger
):
main_logger.log(f"{r.id}: Auction confirmed.")
requests_to_execute.append(r)
else:
main_logger.log(
f"Skipping request {r.id}: unable to verify auction."
)
dump_known_requests(state_file, [r], status="SKIP")
else:
main_logger.log(
f"Skipping request {r.id}: not handling requests with auction."
)
dump_known_requests(state_file, [r], status="SKIP")
elif min_required_payment:
if min_required_payment:
for r in requests_to_check_for_payment:
main_logger.log(f"{r.id}: Confirming payment...")
if confirm_payment(
laconic,
@@ -701,8 +599,8 @@ def command( # noqa: C901
f"Skipping request {r.id}: unable to verify payment."
)
dump_known_requests(state_file, [r], status="UNPAID")
else:
requests_to_execute.append(r)
else:
requests_to_execute = requests_to_check_for_payment
main_logger.log(
"Found %d unsatisfied request(s) to process." % len(requests_to_execute)
@@ -727,7 +625,7 @@ def command( # noqa: C901
)
run_log_file = open(run_log_file_path, "wt")
run_reg_client = LaconicRegistryClient(
laconic_config, log_file=run_log_file, mutex_lock_file=registry_lock_file
laconic_config, log_file=run_log_file
)
build_logger = TimedLogger(run_id, run_log_file)
@@ -739,7 +637,6 @@ def command( # noqa: C901
record_namespace_deployments,
record_namespace_dns,
dns_suffix,
ip,
os.path.abspath(deployment_parent_dir),
kube_config,
image_registry,
@@ -1,220 +0,0 @@
# Copyright ©2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import sys
import json
import click
from stack_orchestrator.deploy.webapp.util import (
AttrDict,
LaconicRegistryClient,
TimedLogger,
load_known_requests,
AUCTION_KIND_PROVIDER,
AuctionStatus,
)
def process_app_deployment_auction(
ctx,
laconic: LaconicRegistryClient,
request,
current_status,
reveal_file_path,
bid_amount,
logger,
):
# Fetch auction details
auction_id = request.attributes.auction
auction = laconic.get_auction(auction_id)
if not auction:
raise Exception(f"Unable to locate auction: {auction_id}")
# Check auction kind
if auction.kind != AUCTION_KIND_PROVIDER:
raise Exception(f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction.kind}")
if current_status == "PENDING":
# Skip if pending auction not in commit state
if auction.status != AuctionStatus.COMMIT:
logger.log(f"Skipping pending request, auction {auction_id} status: {auction.status}")
return "SKIP", ""
# Check max_price
bid_amount_int = int(bid_amount)
max_price_int = int(auction.maxPrice.quantity)
if max_price_int < bid_amount_int:
logger.log(f"Skipping auction {auction_id} with max_price ({max_price_int}) less than bid_amount ({bid_amount_int})")
return "SKIP", ""
# Bid on the auction
reveal_file_path = laconic.commit_bid(auction_id, bid_amount_int)
logger.log(f"Commited bid on auction {auction_id} with amount {bid_amount_int}")
return "COMMIT", reveal_file_path
if current_status == "COMMIT":
# Return if auction still in commit state
if auction.status == AuctionStatus.COMMIT:
logger.log(f"Auction {auction_id} status: {auction.status}")
return current_status, reveal_file_path
# Reveal bid
if auction.status == AuctionStatus.REVEAL:
laconic.reveal_bid(auction_id, reveal_file_path)
logger.log(f"Revealed bid on auction {auction_id}")
return "REVEAL", reveal_file_path
raise Exception(f"Unexpected auction {auction_id} status: {auction.status}")
if current_status == "REVEAL":
# Return if auction still in reveal state
if auction.status == AuctionStatus.REVEAL:
logger.log(f"Auction {auction_id} status: {auction.status}")
return current_status, reveal_file_path
# Return if auction is completed
if auction.status == AuctionStatus.COMPLETED:
logger.log(f"Auction {auction_id} completed")
return "COMPLETED", ""
raise Exception(f"Unexpected auction {auction_id} status: {auction.status}")
raise Exception(f"Got request with unexpected status: {current_status}")
def dump_known_auction_requests(filename, requests, status="SEEN"):
if not filename:
return
known_requests = load_known_requests(filename)
for r in requests:
known_requests[r.id] = {"revealFile": r.revealFile, "status": status}
with open(filename, "w") as f:
json.dump(known_requests, f)
@click.command()
@click.option(
"--laconic-config", help="Provide a config file for laconicd", required=True
)
@click.option(
"--state-file",
help="File to store state about previously seen auction requests.",
required=True,
)
@click.option(
"--bid-amount",
help="Bid to place on application deployment auctions (in alnt)",
required=True,
)
@click.option(
"--registry-lock-file", help="File path to use for registry mutex lock", default=None
)
@click.option(
"--dry-run", help="Don't do anything, just report what would be done.", is_flag=True
)
@click.pass_context
def command(
ctx,
laconic_config,
state_file,
bid_amount,
registry_lock_file,
dry_run,
):
if int(bid_amount) < 0:
print("--bid-amount cannot be less than 0", file=sys.stderr)
sys.exit(2)
logger = TimedLogger(file=sys.stderr)
try:
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file)
auctions_requests = laconic.app_deployment_auctions()
previous_requests = {}
logger.log(f"Loading known auctions from {state_file}...")
previous_requests = load_known_requests(state_file)
# Process new requests first
auctions_requests.sort(key=lambda r: r.createTime)
auctions_requests.reverse()
requests_to_execute = []
for r in auctions_requests:
logger.log(f"BEGIN: Examining request {r.id}")
result_status = "PENDING"
reveal_file_path = ""
try:
application = r.attributes.application
# Handle already seen requests
if r.id in previous_requests:
# If it's not in commit or reveal status, skip the request as we've already seen it
current_status = previous_requests[r.id].get("status", "")
result_status = current_status
if current_status not in ["COMMIT", "REVEAL"]:
logger.log(f"Skipping request {r.id}, we've already seen it.")
continue
reveal_file_path = previous_requests[r.id].get("revealFile", "")
logger.log(f"Found existing auction request {r.id} for application {application}, status {current_status}.")
else:
# It's a fresh request, check application record
app = laconic.get_record(application)
if not app:
logger.log(f"Skipping request {r.id}, cannot locate app.")
result_status = "ERROR"
continue
logger.log(f"Found pending auction request {r.id} for application {application}.")
# Add requests to be processed
requests_to_execute.append((r, result_status, reveal_file_path))
except Exception as e:
result_status = "ERROR"
logger.log(f"ERROR: examining request {r.id}: " + str(e))
finally:
logger.log(f"DONE: Examining request {r.id} with result {result_status}.")
if result_status in ["ERROR"]:
dump_known_auction_requests(state_file, [AttrDict({"id": r.id, "revealFile": reveal_file_path})], result_status)
logger.log(f"Found {len(requests_to_execute)} request(s) to process.")
if not dry_run:
for r, current_status, reveal_file_path in requests_to_execute:
logger.log(f"Processing {r.id}: BEGIN")
result_status = "ERROR"
try:
result_status, reveal_file_path = process_app_deployment_auction(
ctx,
laconic,
r,
current_status,
reveal_file_path,
bid_amount,
logger,
)
except Exception as e:
logger.log(f"ERROR {r.id}:" + str(e))
finally:
logger.log(f"Processing {r.id}: END - {result_status}")
dump_known_auction_requests(state_file, [AttrDict({"id": r.id, "revealFile": reveal_file_path})], result_status)
except Exception as e:
logger.log("UNCAUGHT ERROR:" + str(e))
raise e
@@ -1,124 +0,0 @@
# Copyright ©2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import sys
import click
import yaml
from stack_orchestrator.deploy.webapp.util import (
AUCTION_KIND_PROVIDER,
TOKEN_DENOM,
LaconicRegistryClient,
)
def fatal(msg: str):
print(msg, file=sys.stderr)
sys.exit(1)
@click.command()
@click.option(
"--laconic-config", help="Provide a config file for laconicd", required=True
)
@click.option(
"--app",
help="The LRN of the application to deploy.",
required=True,
)
@click.option(
"--commits-duration",
help="Auction commits duration (in seconds) (default: 600).",
default=600,
)
@click.option(
"--reveals-duration",
help="Auction reveals duration (in seconds) (default: 600).",
default=600,
)
@click.option(
"--commit-fee",
help="Auction bid commit fee (in alnt) (default: 100000).",
default=100000,
)
@click.option(
"--reveal-fee",
help="Auction bid reveal fee (in alnt) (default: 100000).",
default=100000,
)
@click.option(
"--max-price",
help="Max acceptable bid price (in alnt).",
required=True,
)
@click.option(
"--num-providers",
help="Max acceptable bid price (in alnt).",
required=True,
)
@click.option(
"--dry-run",
help="Don't publish anything, just report what would be done.",
is_flag=True,
)
@click.pass_context
def command(
ctx,
laconic_config,
app,
commits_duration,
reveals_duration,
commit_fee,
reveal_fee,
max_price,
num_providers,
dry_run,
):
laconic = LaconicRegistryClient(laconic_config)
app_record = laconic.get_record(app)
if not app_record:
fatal(f"Unable to locate app: {app}")
provider_auction_params = {
"kind": AUCTION_KIND_PROVIDER,
"commits_duration": commits_duration,
"reveals_duration": reveals_duration,
"denom": TOKEN_DENOM,
"commit_fee": commit_fee,
"reveal_fee": reveal_fee,
"max_price": max_price,
"num_providers": num_providers,
}
auction_id = laconic.create_deployment_auction(provider_auction_params)
print("Deployment auction created:", auction_id)
if not auction_id:
fatal("Unable to create a provider auction")
deployment_auction = {
"record": {
"type": "ApplicationDeploymentAuction",
"application": app,
"auction": auction_id,
}
}
if dry_run:
print(yaml.dump(deployment_auction))
return
# Publish the deployment auction record
laconic.publish(deployment_auction)
@@ -76,7 +76,6 @@ def command( # noqa: C901
"name": hostname,
"publicKey": pub_key,
"paymentAddress": payment_address,
"deployerVersion": "1.0.0",
}
}
@@ -1,77 +0,0 @@
from functools import wraps
import os
import time
# Define default file path for the lock
DEFAULT_LOCK_FILE_PATH = "/tmp/registry_mutex_lock_file"
LOCK_TIMEOUT = 30
LOCK_RETRY_INTERVAL = 3
def acquire_lock(client, lock_file_path, timeout):
# Lock alreay acquired by the current client
if client.mutex_lock_acquired:
return
while True:
try:
# Check if lock file exists and is potentially stale
if os.path.exists(lock_file_path):
with open(lock_file_path, 'r') as lock_file:
timestamp = float(lock_file.read().strip())
# If lock is stale, remove the lock file
if time.time() - timestamp > timeout:
print(f"Stale lock detected, removing lock file {lock_file_path}")
os.remove(lock_file_path)
else:
print(f"Lock file {lock_file_path} exists and is recent, waiting...")
time.sleep(LOCK_RETRY_INTERVAL)
continue
# Try to create a new lock file with the current timestamp
fd = os.open(lock_file_path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
with os.fdopen(fd, 'w') as lock_file:
lock_file.write(str(time.time()))
client.mutex_lock_acquired = True
print(f"Registry lock acquired, {lock_file_path}")
# Lock successfully acquired
return
except FileExistsError:
print(f"Lock file {lock_file_path} exists, waiting...")
time.sleep(LOCK_RETRY_INTERVAL)
def release_lock(client, lock_file_path):
try:
os.remove(lock_file_path)
client.mutex_lock_acquired = False
print(f"Registry lock released, {lock_file_path}")
except FileNotFoundError:
# Lock file already removed
pass
def registry_mutex():
def decorator(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
lock_file_path = DEFAULT_LOCK_FILE_PATH
if self.mutex_lock_file:
lock_file_path = self.mutex_lock_file
# Acquire the lock before running the function
acquire_lock(self, lock_file_path, LOCK_TIMEOUT)
try:
return func(self, *args, **kwargs)
finally:
# Release the lock after the function completes
release_lock(self, lock_file_path)
return wrapper
return decorator
@@ -3,6 +3,7 @@
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
import base64
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -16,7 +17,6 @@ import shutil
import sys
import tempfile
from datetime import datetime
import base64
import gnupg
import click
@@ -24,8 +24,6 @@ import requests
import yaml
from stack_orchestrator.deploy.webapp.util import (
AUCTION_KIND_PROVIDER,
AuctionStatus,
LaconicRegistryClient,
)
from dotenv import dotenv_values
@@ -45,13 +43,10 @@ def fatal(msg: str):
help="The LRN of the application to deploy.",
required=True,
)
@click.option(
"--auction-id",
help="Deployment auction id. Can be used instead of deployer and payment.",
)
@click.option(
"--deployer",
help="The LRN of the deployer to process this request.",
required=True,
)
@click.option("--env-file", help="environment file for webapp")
@click.option("--config-ref", help="The ref of an existing config upload to use.")
@@ -69,11 +64,10 @@ def fatal(msg: str):
is_flag=True,
)
@click.pass_context
def command( # noqa: C901
def command(
ctx,
laconic_config,
app,
auction_id,
deployer,
env_file,
config_ref,
@@ -81,135 +75,58 @@ def command( # noqa: C901
use_payment,
dns,
dry_run,
):
if auction_id and deployer:
print("Cannot specify both --auction-id and --deployer", file=sys.stderr)
sys.exit(2)
): # noqa: C901
tempdir = tempfile.mkdtemp()
try:
laconic = LaconicRegistryClient(laconic_config)
if not auction_id and not deployer:
print("Must specify either --auction-id or --deployer", file=sys.stderr)
sys.exit(2)
app_record = laconic.get_record(app)
if not app_record:
fatal(f"Unable to locate app: {app}")
if auction_id and (make_payment or use_payment):
print("Cannot specify --auction-id with --make-payment or --use-payment", file=sys.stderr)
sys.exit(2)
if env_file and config_ref:
fatal("Cannot use --env-file and --config-ref at the same time.")
laconic = LaconicRegistryClient(laconic_config)
app_record = laconic.get_record(app)
if not app_record:
fatal(f"Unable to locate app: {app}")
# Deployers to send requests to
deployer_records = []
auction = None
auction_winners = None
if auction_id:
# Fetch auction record for given auction
auction_records_by_id = laconic.app_deployment_auctions({"auction": auction_id})
if len(auction_records_by_id) == 0:
fatal(f"Unable to locate record for auction: {auction_id}")
# Cross check app against application in the auction record
auction_app = auction_records_by_id[0].attributes.application
if auction_app != app:
fatal(f"Requested application {app} does not match application from auction record {auction_app}")
# Fetch auction details
auction = laconic.get_auction(auction_id)
if not auction:
fatal(f"Unable to locate auction: {auction_id}")
# Check auction owner
if auction.ownerAddress != laconic.whoami().address:
fatal(f"Auction {auction_id} owner mismatch")
# Check auction kind
if auction.kind != AUCTION_KIND_PROVIDER:
fatal(f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction.kind}")
# Check auction status
if auction.status != AuctionStatus.COMPLETED:
fatal(f"Auction {auction_id} not completed yet, status {auction.status}")
# Check that winner list is not empty
if len(auction.winnerAddresses) == 0:
fatal(f"Auction {auction_id} has no winners")
auction_winners = auction.winnerAddresses
# Get deployer record for all the auction winners
for auction_winner in auction_winners:
# TODO: Match auction winner address with provider address?
deployer_records_by_owner = laconic.webapp_deployers({"paymentAddress": auction_winner})
if len(deployer_records_by_owner) == 0:
print(f"WARNING: Unable to locate deployer for auction winner {auction_winner}")
# Take first record with name set
target_deployer_record = deployer_records_by_owner[0]
for r in deployer_records_by_owner:
if len(r.names) > 0:
target_deployer_record = r
break
deployer_records.append(target_deployer_record)
else:
deployer_record = laconic.get_record(deployer)
if not deployer_record:
fatal(f"Unable to locate deployer: {deployer}")
deployer_records.append(deployer_record)
if env_file and config_ref:
fatal("Cannot use --env-file and --config-ref at the same time.")
# Create and send request to each deployer
deployment_requests = []
for deployer_record in deployer_records:
# Upload config to deployers if env_file is passed
# If env_file
if env_file:
tempdir = tempfile.mkdtemp()
try:
gpg = gnupg.GPG(gnupghome=tempdir)
gpg = gnupg.GPG(gnupghome=tempdir)
# Import the deployer's public key
result = gpg.import_keys(
base64.b64decode(deployer_record.attributes.publicKey)
)
if 1 != result.imported:
fatal("Failed to import deployer's public key.")
# Import the deployer's public key
result = gpg.import_keys(
base64.b64decode(deployer_record.attributes.publicKey)
)
if 1 != result.imported:
fatal("Failed to import deployer's public key.")
recip = gpg.list_keys()[0]["uids"][0]
recip = gpg.list_keys()[0]["uids"][0]
# Wrap the config
config = {
# Include account (and payment?) details
"authorized": [laconic.whoami().address],
"config": {"env": dict(dotenv_values(env_file))},
}
serialized = yaml.dump(config)
# Wrap the config
config = {
# Include account (and payment?) details
"authorized": [laconic.whoami().address],
"config": {"env": dict(dotenv_values(env_file))},
}
serialized = yaml.dump(config)
# Encrypt
result = gpg.encrypt(serialized, recip, always_trust=True, armor=False)
if not result.ok:
fatal("Failed to encrypt config.")
# Encrypt
result = gpg.encrypt(serialized, recip, always_trust=True, armor=False)
if not result.ok:
fatal("Failed to encrypt config.")
# Upload it to the deployer's API
response = requests.post(
f"{deployer_record.attributes.apiUrl}/upload/config",
data=result.data,
headers={"Content-Type": "application/octet-stream"},
)
if not response.ok:
response.raise_for_status()
# Upload it to the deployer's API
response = requests.post(
f"{deployer_record.attributes.apiUrl}/upload/config",
data=result.data,
headers={"Content-Type": "application/octet-stream"},
)
if not response.ok:
response.raise_for_status()
config_ref = response.json()["id"]
finally:
shutil.rmtree(tempdir, ignore_errors=True)
target_deployer = deployer
if (not deployer) and len(deployer_record.names):
target_deployer = deployer_record.names[0]
config_ref = response.json()["id"]
deployment_request = {
"record": {
@@ -217,14 +134,11 @@ def command( # noqa: C901
"application": app,
"version": "1.0.0",
"name": f"{app_record.attributes.name}@{app_record.attributes.version}",
"deployer": target_deployer,
"deployer": deployer,
"meta": {"when": str(datetime.utcnow())},
}
}
if auction_id:
deployment_request["record"]["auction"] = auction_id
if config_ref:
deployment_request["record"]["config"] = {"ref": config_ref}
@@ -251,12 +165,11 @@ def command( # noqa: C901
elif use_payment:
deployment_request["record"]["payment"] = use_payment
deployment_requests.append(deployment_request)
# Send all requests
for deployment_request in deployment_requests:
if dry_run:
print(yaml.dump(deployment_request))
continue
return
# Send the request
laconic.publish(deployment_request)
finally:
shutil.rmtree(tempdir, ignore_errors=True)
@@ -1,106 +0,0 @@
# Copyright ©2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import sys
import click
import yaml
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient)
def fatal(msg: str):
print(msg, file=sys.stderr)
sys.exit(1)
@click.command()
@click.option(
"--laconic-config", help="Provide a config file for laconicd", required=True
)
@click.option(
"--deployer",
help="The LRN of the deployer to process this request.",
required=True
)
@click.option(
"--deployment",
help="Deployment record (ApplicationDeploymentRecord) id of the deployment to remove.",
required=True,
)
@click.option(
"--make-payment",
help="The payment to make (in alnt). The value should be a number or 'auto' to use the deployer's minimum required payment.",
)
@click.option(
"--use-payment", help="The TX id of an existing, unused payment", default=None
)
@click.option(
"--dry-run",
help="Don't publish anything, just report what would be done.",
is_flag=True,
)
@click.pass_context
def command(
ctx,
laconic_config,
deployer,
deployment,
make_payment,
use_payment,
dry_run,
):
if make_payment and use_payment:
fatal("Cannot use --make-payment and --use-payment at the same time.")
laconic = LaconicRegistryClient(laconic_config)
deployer_record = laconic.get_record(deployer)
if not deployer_record:
fatal(f"Unable to locate deployer: {deployer}")
undeployment_request = {
"record": {
"type": "ApplicationDeploymentRemovalRequest",
"version": "1.0.0",
"deployer": deployer,
"deployment": deployment,
}
}
if make_payment:
amount = 0
if dry_run:
undeployment_request["record"]["payment"] = "DRY_RUN"
elif "auto" == make_payment:
if "minimumPayment" in deployer_record.attributes:
amount = int(
deployer_record.attributes.minimumPayment.replace("alnt", "")
)
else:
amount = make_payment
if amount:
receipt = laconic.send_tokens(
deployer_record.attributes.paymentAddress, amount
)
undeployment_request["record"]["payment"] = receipt.tx.hash
print("Payment TX:", receipt.tx.hash)
elif use_payment:
undeployment_request["record"]["payment"] = use_payment
if dry_run:
print(yaml.dump(undeployment_request))
return
laconic.publish(undeployment_request)
@@ -178,9 +178,6 @@ def dump_known_requests(filename, requests):
"my payment address are examined).",
is_flag=True,
)
@click.option(
"--registry-lock-file", help="File path to use for registry mutex lock", default=None
)
@click.pass_context
def command( # noqa: C901
ctx,
@@ -198,7 +195,6 @@ def command( # noqa: C901
min_required_payment,
lrn,
all_requests,
registry_lock_file,
):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
@@ -216,7 +212,7 @@ def command( # noqa: C901
include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file)
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr)
deployer_record = laconic.get_record(lrn, require=True)
payment_address = deployer_record.attributes.paymentAddress
main_logger.log(f"Payment address: {payment_address}")
@@ -315,7 +311,6 @@ def command( # noqa: C901
main_logger.log(f"ERROR examining {r.id}: {e}")
requests_to_execute = []
# TODO: Handle requests with auction
if min_required_payment:
for r in requests_to_check_for_payment:
main_logger.log(f"{r.id}: Confirming payment...")
+111 -306
View File
@@ -21,25 +21,9 @@ import random
import subprocess
import sys
import tempfile
from typing import List
import uuid
import yaml
from enum import Enum
from stack_orchestrator.deploy.webapp.registry_mutex import registry_mutex
class AuctionStatus(str, Enum):
COMMIT = "commit"
REVEAL = "reveal"
COMPLETED = "completed"
EXPIRED = "expired"
TOKEN_DENOM = "alnt"
AUCTION_KIND_PROVIDER = "provider"
class AttrDict(dict):
def __init__(self, *args, **kwargs):
@@ -74,12 +58,6 @@ class TimedLogger:
self.last = datetime.datetime.now()
def load_known_requests(filename):
if filename and os.path.exists(filename):
return json.load(open(filename, "r"))
return {}
def logged_cmd(log_file, *vargs):
result = None
try:
@@ -114,8 +92,76 @@ def is_id(name_or_id: str):
return not is_lrn(name_or_id)
def confirm_payment(laconic, record, payment_address, min_amount, logger):
req_owner = laconic.get_owner(record)
if req_owner == payment_address:
# No need to confirm payment if the sender and recipient are the same account.
return True
if not record.attributes.payment:
logger.log(f"{record.id}: no payment tx info")
return False
tx = laconic.get_tx(record.attributes.payment)
if not tx:
logger.log(f"{record.id}: cannot locate payment tx")
return False
if tx.code != 0:
logger.log(
f"{record.id}: payment tx {tx.hash} was not successful - code: {tx.code}, log: {tx.log}"
)
return False
if tx.sender != req_owner:
logger.log(
f"{record.id}: payment sender {tx.sender} in tx {tx.hash} does not match deployment "
f"request owner {req_owner}"
)
return False
if tx.recipient != payment_address:
logger.log(
f"{record.id}: payment recipient {tx.recipient} in tx {tx.hash} does not match {payment_address}"
)
return False
pay_denom = "".join([i for i in tx.amount if not i.isdigit()])
if pay_denom != "alnt":
logger.log(
f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected payment denomination"
)
return False
pay_amount = int("".join([i for i in tx.amount if i.isdigit()]))
if pay_amount < min_amount:
logger.log(
f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}"
)
return False
# Check if the payment was already used on a
used = laconic.app_deployments(
{"deployer": payment_address, "payment": tx.hash}, all=True
)
if len(used):
logger.log(f"{record.id}: payment {tx.hash} already used on deployment {used}")
return False
used = laconic.app_deployment_removals(
{"deployer": payment_address, "payment": tx.hash}, all=True
)
if len(used):
logger.log(
f"{record.id}: payment {tx.hash} already used on deployment removal {used}"
)
return False
return True
class LaconicRegistryClient:
def __init__(self, config_file, log_file=None, mutex_lock_file=None):
def __init__(self, config_file, log_file=None):
self.config_file = config_file
self.log_file = log_file
self.cache = AttrDict(
@@ -126,9 +172,6 @@ class LaconicRegistryClient:
}
)
self.mutex_lock_file = mutex_lock_file
self.mutex_lock_acquired = False
def whoami(self, refresh=False):
if not refresh and "whoami" in self.cache:
return self.cache["whoami"]
@@ -327,34 +370,6 @@ class LaconicRegistryClient:
if require:
raise Exception("Cannot locate tx:", hash)
def get_auction(self, auction_id, require=False):
args = [
"laconic",
"-c",
self.config_file,
"registry",
"auction",
"get",
"--id",
auction_id,
]
results = None
try:
results = [
AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
]
except: # noqa: E722
pass
if results and len(results):
return results[0]
if require:
raise Exception("Cannot locate auction:", auction_id)
return None
def app_deployment_requests(self, criteria=None, all=True):
if criteria is None:
criteria = {}
@@ -383,21 +398,6 @@ class LaconicRegistryClient:
criteria["type"] = "ApplicationDeploymentRemovalRecord"
return self.list_records(criteria, all)
def webapp_deployers(self, criteria=None, all=True):
if criteria is None:
criteria = {}
criteria = criteria.copy()
criteria["type"] = "WebappDeployer"
return self.list_records(criteria, all)
def app_deployment_auctions(self, criteria=None, all=True):
if criteria is None:
criteria = {}
criteria = criteria.copy()
criteria["type"] = "ApplicationDeploymentAuction"
return self.list_records(criteria, all)
@registry_mutex()
def publish(self, record, names=None):
if names is None:
names = []
@@ -428,7 +428,6 @@ class LaconicRegistryClient:
finally:
logged_cmd(self.log_file, "rm", "-rf", tmpdir)
@registry_mutex()
def set_name(self, name, record_id):
logged_cmd(
self.log_file,
@@ -442,7 +441,6 @@ class LaconicRegistryClient:
record_id,
)
@registry_mutex()
def delete_name(self, name):
logged_cmd(
self.log_file,
@@ -455,7 +453,6 @@ class LaconicRegistryClient:
name,
)
@registry_mutex()
def send_tokens(self, address, amount, type="alnt"):
args = [
"laconic",
@@ -474,67 +471,6 @@ class LaconicRegistryClient:
return AttrDict(json.loads(logged_cmd(self.log_file, *args)))
@registry_mutex()
def create_deployment_auction(self, auction):
args = [
"laconic",
"-c",
self.config_file,
"registry",
"auction",
"create",
"--kind",
auction["kind"],
"--commits-duration",
str(auction["commits_duration"]),
"--reveals-duration",
str(auction["reveals_duration"]),
"--denom",
auction["denom"],
"--commit-fee",
str(auction["commit_fee"]),
"--reveal-fee",
str(auction["reveal_fee"]),
"--max-price",
str(auction["max_price"]),
"--num-providers",
str(auction["num_providers"])
]
return json.loads(logged_cmd(self.log_file, *args))["auctionId"]
@registry_mutex()
def commit_bid(self, auction_id, amount, type="alnt"):
args = [
"laconic",
"-c",
self.config_file,
"registry",
"auction",
"bid",
"commit",
auction_id,
str(amount),
type,
]
return json.loads(logged_cmd(self.log_file, *args))["reveal_file"]
@registry_mutex()
def reveal_bid(self, auction_id, reveal_file_path):
logged_cmd(
self.log_file,
"laconic",
"-c",
self.config_file,
"registry",
"auction",
"bid",
"reveal",
auction_id,
reveal_file_path,
)
def file_hash(filename):
return hashlib.sha1(open(filename).read().encode()).hexdigest()
@@ -686,12 +622,10 @@ def publish_deployment(
laconic: LaconicRegistryClient,
app_record,
deploy_record,
deployment_lrns,
existing_dns_records_by_lrns,
dns_lrns: List[str],
dns_record_namespace,
deployment_lrn,
dns_record,
dns_lrn,
deployment_dir,
dns_value=None,
app_deployment_request=None,
webapp_deployer_record=None,
logger=None,
@@ -703,97 +637,73 @@ def publish_deployment(
int(deploy_record.attributes.version.split(".")[-1]) + 1
)
dns_ids = []
for dns_lrn in dns_lrns:
dns_record = existing_dns_records_by_lrns[dns_lrn]
if not dns_record:
dns_ver = "0.0.1"
else:
dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1)
fqdn = dns_lrn.removeprefix(f"{dns_record_namespace}/")
uniq = uuid.uuid4()
new_dns_record = {
"record": {
"type": "DnsRecord",
"version": dns_ver,
"name": fqdn,
"resource_type": "A",
"meta": {"so": uniq.hex},
}
}
if app_deployment_request:
new_dns_record["record"]["request"] = app_deployment_request.id
if dns_value:
new_dns_record["record"]["value"] = dns_value
if logger:
logger.log("Publishing DnsRecord.")
dns_id = laconic.publish(new_dns_record, [dns_lrn])
dns_ids.append(dns_id)
if not dns_record:
dns_ver = "0.0.1"
else:
dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1)
spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml")))
last_fqdn = spec["network"]["http-proxy"][-1]["host-name"]
last_dns_id = dns_ids[-1]
fqdn = spec["network"]["http-proxy"][0]["host-name"]
uniq = uuid.uuid4()
new_dns_record = {
"record": {
"type": "DnsRecord",
"version": dns_ver,
"name": fqdn,
"resource_type": "A",
"meta": {"so": uniq.hex},
}
}
if app_deployment_request:
new_dns_record["record"]["request"] = app_deployment_request.id
if logger:
logger.log("Publishing DnsRecord.")
dns_id = laconic.publish(new_dns_record, [dns_lrn])
new_deployment_record = {
"record": {
"type": "ApplicationDeploymentRecord",
"version": deploy_ver,
"url": f"https://{last_fqdn}",
"url": f"https://{fqdn}",
"name": app_record.attributes.name,
"application": app_record.id,
"dns": last_dns_id,
"dns": dns_id,
"meta": {
"config": file_hash(os.path.join(deployment_dir, "config.env")),
"so": uniq.hex,
},
}
}
if app_deployment_request:
new_deployment_record["record"]["request"] = app_deployment_request.id
# Set auction or payment id from request
if app_deployment_request.attributes.auction:
new_deployment_record["record"]["auction"] = app_deployment_request.attributes.auction
elif app_deployment_request.attributes.payment:
new_deployment_record["record"]["payment"] = app_deployment_request.attributes.payment
if app_deployment_request.attributes.payment:
new_deployment_record["record"][
"payment"
] = app_deployment_request.attributes.payment
if webapp_deployer_record:
new_deployment_record["record"]["deployer"] = webapp_deployer_record.names[0]
if logger:
logger.log("Publishing ApplicationDeploymentRecord.")
deployment_id = laconic.publish(new_deployment_record, deployment_lrns)
return {"dns": dns_ids, "deployment": deployment_id}
deployment_id = laconic.publish(new_deployment_record, [deployment_lrn])
return {"dns": dns_id, "deployment": deployment_id}
def get_requested_names(app_deployment_request):
request_dns = app_deployment_request.attributes.dns
return request_dns.split(",") if request_dns else []
def hostnames_for_deployment_request(app_deployment_request, laconic):
requested_names = get_requested_names(app_deployment_request)
if len(requested_names) == 0:
def hostname_for_deployment_request(app_deployment_request, laconic):
dns_name = app_deployment_request.attributes.dns
if not dns_name:
app = laconic.get_record(
app_deployment_request.attributes.application, require=True
)
return [generate_hostname_for_app(app)]
dns_names = []
for requested_name in requested_names:
dns_name = requested_name
if dns_name.startswith("lrn://"):
record = laconic.get_record(dns_name, require=True)
dns_name = record.attributes.name
dns_names.append(dns_name)
return dns_names
dns_name = generate_hostname_for_app(app)
elif dns_name.startswith("lrn://"):
record = laconic.get_record(dns_name, require=True)
dns_name = record.attributes.name
return dns_name
def generate_hostname_for_app(app):
@@ -820,108 +730,3 @@ def skip_by_tag(r, include_tags, exclude_tags):
return True
return False
def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min_amount, logger):
req_owner = laconic.get_owner(record)
if req_owner == payment_address:
# No need to confirm payment if the sender and recipient are the same account.
return True
if not record.attributes.payment:
logger.log(f"{record.id}: no payment tx info")
return False
tx = laconic.get_tx(record.attributes.payment)
if not tx:
logger.log(f"{record.id}: cannot locate payment tx")
return False
if tx.code != 0:
logger.log(
f"{record.id}: payment tx {tx.hash} was not successful - code: {tx.code}, log: {tx.log}"
)
return False
if tx.sender != req_owner:
logger.log(
f"{record.id}: payment sender {tx.sender} in tx {tx.hash} does not match deployment "
f"request owner {req_owner}"
)
return False
if tx.recipient != payment_address:
logger.log(
f"{record.id}: payment recipient {tx.recipient} in tx {tx.hash} does not match {payment_address}"
)
return False
pay_denom = "".join([i for i in tx.amount if not i.isdigit()])
if pay_denom != "alnt":
logger.log(
f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected payment denomination"
)
return False
pay_amount = int("".join([i for i in tx.amount if i.isdigit()]))
if pay_amount < min_amount:
logger.log(
f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}"
)
return False
# Check if the payment was already used on a deployment
used = laconic.app_deployments(
{"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
)
if len(used):
# Fetch the app name from request record
used_request = laconic.get_record(used[0].attributes.request, require=True)
# Check that payment was used for deployment of same application
if record.attributes.application != used_request.attributes.application:
logger.log(f"{record.id}: payment {tx.hash} already used on a different application deployment {used}")
return False
used = laconic.app_deployment_removals(
{"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
)
if len(used):
logger.log(
f"{record.id}: payment {tx.hash} already used on deployment removal {used}"
)
return False
return True
def confirm_auction(laconic: LaconicRegistryClient, record, deployer_lrn, payment_address, logger):
auction_id = record.attributes.auction
auction = laconic.get_auction(auction_id)
# Fetch auction record for given auction
auction_records_by_id = laconic.app_deployment_auctions({"auction": auction_id})
if len(auction_records_by_id) == 0:
logger.log(f"{record.id}: unable to locate record for auction {auction_id}")
return False
# Cross check app against application in the auction record
requested_app = laconic.get_record(record.attributes.application, require=True)
auction_app = laconic.get_record(auction_records_by_id[0].attributes.application, require=True)
if requested_app.id != auction_app.id:
logger.log(
f"{record.id}: requested application {record.attributes.application} does not match application from "
f"auction record {auction_records_by_id[0].attributes.application}"
)
return False
if not auction:
logger.log(f"{record.id}: unable to locate auction {auction_id}")
return False
# Check if the deployer payment address is in auction winners list
if payment_address not in auction.winnerAddresses:
logger.log(f"{record.id}: deployer payment address not in auction winners.")
return False
return True
+1 -7
View File
@@ -26,10 +26,7 @@ from stack_orchestrator.deploy.webapp import (run_webapp,
deploy_webapp_from_registry,
undeploy_webapp_from_registry,
publish_webapp_deployer,
publish_deployment_auction,
handle_deployment_auction,
request_webapp_deployment,
request_webapp_undeployment)
request_webapp_deployment)
from stack_orchestrator.deploy import deploy
from stack_orchestrator import version
from stack_orchestrator.deploy import deployment
@@ -67,10 +64,7 @@ cli.add_command(deploy_webapp.command, "deploy-webapp")
cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry")
cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry")
cli.add_command(publish_webapp_deployer.command, "publish-deployer-to-registry")
cli.add_command(publish_deployment_auction.command, "publish-deployment-auction")
cli.add_command(handle_deployment_auction.command, "handle-deployment-auction")
cli.add_command(request_webapp_deployment.command, "request-webapp-deployment")
cli.add_command(request_webapp_undeployment.command, "request-webapp-undeployment")
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
cli.add_command(deploy.command, "deploy-system")
cli.add_command(deployment.command, "deployment")