From 0c47da42feb2deec2bfc30c3a00bf15db1eeeb84 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Mon, 21 Oct 2024 07:02:06 +0000 Subject: [PATCH] Integrate SP auctions in webapp deployment flow (#950) Part of [Service provider auctions for web deployments](https://www.notion.so/Service-provider-auctions-for-web-deployments-104a6b22d47280dbad51d28aa3a91d75) and https://git.vdb.to/cerc-io/stack-orchestrator/issues/948 - Add a command `publish-deployment-auction` to create and publish an app deployment auction - Add a command `handle-deployment-auction` to handle auctions on deployer side - Update `request-webapp-deployment` command to allow using an auction id in deployment requests - Update `deploy-webapp-from-registry` command to handle deployment requests with auction - Add a command `request-webapp-undeployment` to request an application undeployment Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/950 Reviewed-by: ashwin Co-authored-by: Prathamesh Musale Co-committed-by: Prathamesh Musale --- .../data/stacks/package-registry/README.md | 22 ++ .../webapp/deploy_webapp_from_registry.py | 44 ++- .../webapp/handle_deployment_auction.py | 216 ++++++++++++ .../webapp/publish_deployment_auction.py | 124 +++++++ .../webapp/request_webapp_deployment.py | 179 +++++++--- .../webapp/request_webapp_undeployment.py | 106 ++++++ .../webapp/undeploy_webapp_from_registry.py | 1 + stack_orchestrator/deploy/webapp/util.py | 322 ++++++++++++++---- stack_orchestrator/main.py | 8 +- 9 files changed, 893 insertions(+), 129 deletions(-) create mode 100644 stack_orchestrator/deploy/webapp/handle_deployment_auction.py create mode 100644 stack_orchestrator/deploy/webapp/publish_deployment_auction.py create mode 100644 stack_orchestrator/deploy/webapp/request_webapp_undeployment.py diff --git a/stack_orchestrator/data/stacks/package-registry/README.md b/stack_orchestrator/data/stacks/package-registry/README.md index 98630804..469cd8a4 100644 --- a/stack_orchestrator/data/stacks/package-registry/README.md +++ b/stack_orchestrator/data/stacks/package-registry/README.md @@ -27,3 +27,25 @@ The Package Registry Stack supports a build environment that requires a package ``` * The local gitea registry can now be accessed at (the username and password can be taken from the deployment logs) + +* Configure the hostname `gitea.local`: + + Update `/etc/hosts`: + + ```bash + sudo nano /etc/hosts + + # Add the following line + 127.0.0.1 gitea.local + ``` + + Check resolution: + + ```bash + ping gitea.local + + PING gitea.local (127.0.0.1) 56(84) bytes of data. + 64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.147 ms + 64 bytes from localhost (127.0.0.1): icmp_seq=2 ttl=64 time=0.033 ms + ... + ``` diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index aa3a4d8f..7fa2e255 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -33,6 +33,7 @@ from stack_orchestrator.deploy.webapp.util import ( LaconicRegistryClient, TimedLogger, build_container_image, + confirm_auction, push_container_image, file_hash, deploy_to_k8s, @@ -42,6 +43,7 @@ from stack_orchestrator.deploy.webapp.util import ( match_owner, skip_by_tag, confirm_payment, + load_known_requests, ) @@ -257,12 +259,6 @@ def process_app_deployment_request( logger.log("END - process_app_deployment_request") -def load_known_requests(filename): - if filename and os.path.exists(filename): - return json.load(open(filename, "r")) - return {} - - def dump_known_requests(filename, requests, status="SEEN"): if not filename: return @@ -350,6 +346,12 @@ def dump_known_requests(filename, requests, status="SEEN"): "my payment address are examined).", is_flag=True, ) +@click.option( + "--auction-requests", + help="Handle requests with auction id set (skips payment confirmation).", + is_flag=True, + default=False, +) @click.option( "--config-upload-dir", help="The directory containing uploaded config.", @@ -390,6 +392,7 @@ def command( # noqa: C901 private_key_file, private_key_passphrase, all_requests, + auction_requests, ): if request_id and discover: print("Cannot specify both --request-id and --discover", file=sys.stderr) @@ -582,8 +585,29 @@ def command( # noqa: C901 requests_to_check_for_payment.append(r) requests_to_execute = [] - if min_required_payment: - for r in requests_to_check_for_payment: + for r in requests_to_check_for_payment: + if r.attributes.auction: + if auction_requests: + if confirm_auction( + laconic, + r, + lrn, + payment_address, + main_logger + ): + main_logger.log(f"{r.id}: Auction confirmed.") + requests_to_execute.append(r) + else: + main_logger.log( + f"Skipping request {r.id}: unable to verify auction." + ) + dump_known_requests(state_file, [r], status="SKIP") + else: + main_logger.log( + f"Skipping request {r.id}: not handling requests with auction." + ) + dump_known_requests(state_file, [r], status="SKIP") + elif min_required_payment: main_logger.log(f"{r.id}: Confirming payment...") if confirm_payment( laconic, @@ -599,8 +623,8 @@ def command( # noqa: C901 f"Skipping request {r.id}: unable to verify payment." ) dump_known_requests(state_file, [r], status="UNPAID") - else: - requests_to_execute = requests_to_check_for_payment + else: + requests_to_execute.append(r) main_logger.log( "Found %d unsatisfied request(s) to process." % len(requests_to_execute) diff --git a/stack_orchestrator/deploy/webapp/handle_deployment_auction.py b/stack_orchestrator/deploy/webapp/handle_deployment_auction.py new file mode 100644 index 00000000..380e8b49 --- /dev/null +++ b/stack_orchestrator/deploy/webapp/handle_deployment_auction.py @@ -0,0 +1,216 @@ +# Copyright ©2023 Vulcanize +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys +import json + +import click + +from stack_orchestrator.deploy.webapp.util import ( + AttrDict, + LaconicRegistryClient, + TimedLogger, + load_known_requests, + AUCTION_KIND_PROVIDER, + AuctionStatus, +) + + +def process_app_deployment_auction( + ctx, + laconic: LaconicRegistryClient, + request, + current_status, + reveal_file_path, + bid_amount, + logger, +): + # Fetch auction details + auction_id = request.attributes.auction + auction = laconic.get_auction(auction_id) + if not auction: + raise Exception(f"Unable to locate auction: {auction_id}") + + # Check auction kind + if auction.kind != AUCTION_KIND_PROVIDER: + raise Exception(f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction.kind}") + + if current_status == "PENDING": + # Skip if pending auction not in commit state + if auction.status != AuctionStatus.COMMIT: + logger.log(f"Skipping pending request, auction {auction_id} status: {auction.status}") + return "SKIP", "" + + # Check max_price + bid_amount_int = int(bid_amount) + max_price_int = int(auction.maxPrice.quantity) + if max_price_int < bid_amount_int: + logger.log(f"Skipping auction {auction_id} with max_price ({max_price_int}) less than bid_amount ({bid_amount_int})") + return "SKIP", "" + + # Bid on the auction + reveal_file_path = laconic.commit_bid(auction_id, bid_amount_int) + logger.log(f"Commited bid on auction {auction_id} with amount {bid_amount_int}") + + return "COMMIT", reveal_file_path + + if current_status == "COMMIT": + # Return if auction still in commit state + if auction.status == AuctionStatus.COMMIT: + logger.log(f"Auction {auction_id} status: {auction.status}") + return current_status, reveal_file_path + + # Reveal bid + if auction.status == AuctionStatus.REVEAL: + laconic.reveal_bid(auction_id, reveal_file_path) + logger.log(f"Revealed bid on auction {auction_id}") + + return "REVEAL", reveal_file_path + + raise Exception(f"Unexpected auction {auction_id} status: {auction.status}") + + if current_status == "REVEAL": + # Return if auction still in reveal state + if auction.status == AuctionStatus.REVEAL: + logger.log(f"Auction {auction_id} status: {auction.status}") + return current_status, reveal_file_path + + # Return if auction is completed + if auction.status == AuctionStatus.COMPLETED: + logger.log(f"Auction {auction_id} completed") + return "COMPLETED", "" + + raise Exception(f"Unexpected auction {auction_id} status: {auction.status}") + + raise Exception(f"Got request with unexpected status: {current_status}") + + +def dump_known_auction_requests(filename, requests, status="SEEN"): + if not filename: + return + known_requests = load_known_requests(filename) + for r in requests: + known_requests[r.id] = {"revealFile": r.revealFile, "status": status} + with open(filename, "w") as f: + json.dump(known_requests, f) + + +@click.command() +@click.option( + "--laconic-config", help="Provide a config file for laconicd", required=True +) +@click.option( + "--state-file", + help="File to store state about previously seen auction requests.", + required=True, +) +@click.option( + "--bid-amount", + help="Bid to place on application deployment auctions (in alnt)", + required=True, +) +@click.option( + "--dry-run", help="Don't do anything, just report what would be done.", is_flag=True +) +@click.pass_context +def command( + ctx, + laconic_config, + state_file, + bid_amount, + dry_run, +): + if int(bid_amount) < 0: + print("--bid-amount cannot be less than 0", file=sys.stderr) + sys.exit(2) + + logger = TimedLogger(file=sys.stderr) + + try: + laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr) + auctions_requests = laconic.app_deployment_auctions() + + previous_requests = {} + logger.log(f"Loading known auctions from {state_file}...") + previous_requests = load_known_requests(state_file) + + # Process new requests first + auctions_requests.sort(key=lambda r: r.createTime) + auctions_requests.reverse() + + requests_to_execute = [] + + for r in auctions_requests: + logger.log(f"BEGIN: Examining request {r.id}") + result_status = "PENDING" + reveal_file_path = "" + try: + application = r.attributes.application + + # Handle already seen requests + if r.id in previous_requests: + # If it's not in commit or reveal status, skip the request as we've already seen it + current_status = previous_requests[r.id].get("status", "") + result_status = current_status + if current_status not in ["COMMIT", "REVEAL"]: + logger.log(f"Skipping request {r.id}, we've already seen it.") + continue + + reveal_file_path = previous_requests[r.id].get("revealFile", "") + logger.log(f"Found existing auction request {r.id} for application {application}, status {current_status}.") + else: + # It's a fresh request, check application record + app = laconic.get_record(application) + if not app: + logger.log(f"Skipping request {r.id}, cannot locate app.") + result_status = "ERROR" + continue + + logger.log(f"Found pending auction request {r.id} for application {application}.") + + # Add requests to be processed + requests_to_execute.append((r, result_status, reveal_file_path)) + + except Exception as e: + result_status = "ERROR" + logger.log(f"ERROR: examining request {r.id}: " + str(e)) + finally: + logger.log(f"DONE: Examining request {r.id} with result {result_status}.") + if result_status in ["ERROR"]: + dump_known_auction_requests(state_file, [AttrDict({"id": r.id, "revealFile": reveal_file_path})], result_status) + + logger.log(f"Found {len(requests_to_execute)} request(s) to process.") + + if not dry_run: + for r, current_status, reveal_file_path in requests_to_execute: + logger.log(f"Processing {r.id}: BEGIN") + result_status = "ERROR" + try: + result_status, reveal_file_path = process_app_deployment_auction( + ctx, + laconic, + r, + current_status, + reveal_file_path, + bid_amount, + logger, + ) + except Exception as e: + logger.log(f"ERROR {r.id}:" + str(e)) + finally: + logger.log(f"Processing {r.id}: END - {result_status}") + dump_known_auction_requests(state_file, [AttrDict({"id": r.id, "revealFile": reveal_file_path})], result_status) + except Exception as e: + logger.log("UNCAUGHT ERROR:" + str(e)) + raise e diff --git a/stack_orchestrator/deploy/webapp/publish_deployment_auction.py b/stack_orchestrator/deploy/webapp/publish_deployment_auction.py new file mode 100644 index 00000000..2a05ad17 --- /dev/null +++ b/stack_orchestrator/deploy/webapp/publish_deployment_auction.py @@ -0,0 +1,124 @@ +# Copyright ©2023 Vulcanize +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys + +import click +import yaml + +from stack_orchestrator.deploy.webapp.util import ( + AUCTION_KIND_PROVIDER, + TOKEN_DENOM, + LaconicRegistryClient, +) + + +def fatal(msg: str): + print(msg, file=sys.stderr) + sys.exit(1) + + +@click.command() +@click.option( + "--laconic-config", help="Provide a config file for laconicd", required=True +) +@click.option( + "--app", + help="The LRN of the application to deploy.", + required=True, +) +@click.option( + "--commits-duration", + help="Auction commits duration (in seconds) (default: 600).", + default=600, +) +@click.option( + "--reveals-duration", + help="Auction reveals duration (in seconds) (default: 600).", + default=600, +) +@click.option( + "--commit-fee", + help="Auction bid commit fee (in alnt) (default: 100000).", + default=100000, +) +@click.option( + "--reveal-fee", + help="Auction bid reveal fee (in alnt) (default: 100000).", + default=100000, +) +@click.option( + "--max-price", + help="Max acceptable bid price (in alnt).", + required=True, +) +@click.option( + "--num-providers", + help="Max acceptable bid price (in alnt).", + required=True, +) +@click.option( + "--dry-run", + help="Don't publish anything, just report what would be done.", + is_flag=True, +) +@click.pass_context +def command( + ctx, + laconic_config, + app, + commits_duration, + reveals_duration, + commit_fee, + reveal_fee, + max_price, + num_providers, + dry_run, +): + laconic = LaconicRegistryClient(laconic_config) + + app_record = laconic.get_record(app) + if not app_record: + fatal(f"Unable to locate app: {app}") + + provider_auction_params = { + "kind": AUCTION_KIND_PROVIDER, + "commits_duration": commits_duration, + "reveals_duration": reveals_duration, + "denom": TOKEN_DENOM, + "commit_fee": commit_fee, + "reveal_fee": reveal_fee, + "max_price": max_price, + "num_providers": num_providers, + } + auction_id = laconic.create_auction(provider_auction_params) + print("Deployment auction created:", auction_id) + + if not auction_id: + fatal("Unable to create a provider auction") + + deployment_auction = { + "record": { + "type": "ApplicationDeploymentAuction", + "application": app, + "auction": auction_id, + } + } + + if dry_run: + print(yaml.dump(deployment_auction)) + return + + # Publish the deployment auction record + laconic.publish(deployment_auction) diff --git a/stack_orchestrator/deploy/webapp/request_webapp_deployment.py b/stack_orchestrator/deploy/webapp/request_webapp_deployment.py index 466c4c54..0fb2cff1 100644 --- a/stack_orchestrator/deploy/webapp/request_webapp_deployment.py +++ b/stack_orchestrator/deploy/webapp/request_webapp_deployment.py @@ -3,7 +3,6 @@ # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. -import base64 # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -17,6 +16,7 @@ import shutil import sys import tempfile from datetime import datetime +import base64 import gnupg import click @@ -24,6 +24,8 @@ import requests import yaml from stack_orchestrator.deploy.webapp.util import ( + AUCTION_KIND_PROVIDER, + AuctionStatus, LaconicRegistryClient, ) from dotenv import dotenv_values @@ -43,10 +45,13 @@ def fatal(msg: str): help="The LRN of the application to deploy.", required=True, ) +@click.option( + "--auction-id", + help="Deployment auction id. Can be used instead of deployer and payment.", +) @click.option( "--deployer", help="The LRN of the deployer to process this request.", - required=True, ) @click.option("--env-file", help="environment file for webapp") @click.option("--config-ref", help="The ref of an existing config upload to use.") @@ -64,10 +69,11 @@ def fatal(msg: str): is_flag=True, ) @click.pass_context -def command( +def command( # noqa: C901 ctx, laconic_config, app, + auction_id, deployer, env_file, config_ref, @@ -75,58 +81,135 @@ def command( use_payment, dns, dry_run, -): # noqa: C901 - tempdir = tempfile.mkdtemp() - try: - laconic = LaconicRegistryClient(laconic_config) +): + if auction_id and deployer: + print("Cannot specify both --auction-id and --deployer", file=sys.stderr) + sys.exit(2) - app_record = laconic.get_record(app) - if not app_record: - fatal(f"Unable to locate app: {app}") + if not auction_id and not deployer: + print("Must specify either --auction-id or --deployer", file=sys.stderr) + sys.exit(2) + if auction_id and (make_payment or use_payment): + print("Cannot specify --auction-id with --make-payment or --use-payment", file=sys.stderr) + sys.exit(2) + + if env_file and config_ref: + fatal("Cannot use --env-file and --config-ref at the same time.") + + laconic = LaconicRegistryClient(laconic_config) + + app_record = laconic.get_record(app) + if not app_record: + fatal(f"Unable to locate app: {app}") + + # Deployers to send requests to + deployer_records = [] + + auction = None + auction_winners = None + if auction_id: + # Fetch auction record for given auction + auction_records_by_id = laconic.app_deployment_auctions({"auction": auction_id}) + if len(auction_records_by_id) == 0: + fatal(f"Unable to locate record for auction: {auction_id}") + + # Cross check app against application in the auction record + auction_app = auction_records_by_id[0].attributes.application + if auction_app != app: + fatal(f"Requested application {app} does not match application from auction record {auction_app}") + + # Fetch auction details + auction = laconic.get_auction(auction_id) + if not auction: + fatal(f"Unable to locate auction: {auction_id}") + + # Check auction owner + if auction.ownerAddress != laconic.whoami().address: + fatal(f"Auction {auction_id} owner mismatch") + + # Check auction kind + if auction.kind != AUCTION_KIND_PROVIDER: + fatal(f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction.kind}") + + # Check auction status + if auction.status != AuctionStatus.COMPLETED: + fatal(f"Auction {auction_id} not completed yet, status {auction.status}") + + # Check that winner list is not empty + if len(auction.winnerAddresses) == 0: + fatal(f"Auction {auction_id} has no winners") + + auction_winners = auction.winnerAddresses + + # Get deployer record for all the auction winners + for auction_winner in auction_winners: + # TODO: Match auction winner address with provider address? + deployer_records_by_owner = laconic.webapp_deployers({"paymentAddress": auction_winner}) + if len(deployer_records_by_owner) == 0: + print(f"WARNING: Unable to locate deployer for auction winner {auction_winner}") + + # Take first record with name set + target_deployer_record = deployer_records_by_owner[0] + for r in deployer_records_by_owner: + if len(r.names) > 0: + target_deployer_record = r + break + deployer_records.append(target_deployer_record) + else: deployer_record = laconic.get_record(deployer) if not deployer_record: fatal(f"Unable to locate deployer: {deployer}") - if env_file and config_ref: - fatal("Cannot use --env-file and --config-ref at the same time.") + deployer_records.append(deployer_record) - # If env_file + # Create and send request to each deployer + deployment_requests = [] + for deployer_record in deployer_records: + # Upload config to deployers if env_file is passed if env_file: - gpg = gnupg.GPG(gnupghome=tempdir) + tempdir = tempfile.mkdtemp() + try: + gpg = gnupg.GPG(gnupghome=tempdir) - # Import the deployer's public key - result = gpg.import_keys( - base64.b64decode(deployer_record.attributes.publicKey) - ) - if 1 != result.imported: - fatal("Failed to import deployer's public key.") + # Import the deployer's public key + result = gpg.import_keys( + base64.b64decode(deployer_record.attributes.publicKey) + ) + if 1 != result.imported: + fatal("Failed to import deployer's public key.") - recip = gpg.list_keys()[0]["uids"][0] + recip = gpg.list_keys()[0]["uids"][0] - # Wrap the config - config = { - # Include account (and payment?) details - "authorized": [laconic.whoami().address], - "config": {"env": dict(dotenv_values(env_file))}, - } - serialized = yaml.dump(config) + # Wrap the config + config = { + # Include account (and payment?) details + "authorized": [laconic.whoami().address], + "config": {"env": dict(dotenv_values(env_file))}, + } + serialized = yaml.dump(config) - # Encrypt - result = gpg.encrypt(serialized, recip, always_trust=True, armor=False) - if not result.ok: - fatal("Failed to encrypt config.") + # Encrypt + result = gpg.encrypt(serialized, recip, always_trust=True, armor=False) + if not result.ok: + fatal("Failed to encrypt config.") - # Upload it to the deployer's API - response = requests.post( - f"{deployer_record.attributes.apiUrl}/upload/config", - data=result.data, - headers={"Content-Type": "application/octet-stream"}, - ) - if not response.ok: - response.raise_for_status() + # Upload it to the deployer's API + response = requests.post( + f"{deployer_record.attributes.apiUrl}/upload/config", + data=result.data, + headers={"Content-Type": "application/octet-stream"}, + ) + if not response.ok: + response.raise_for_status() - config_ref = response.json()["id"] + config_ref = response.json()["id"] + finally: + shutil.rmtree(tempdir, ignore_errors=True) + + target_deployer = deployer + if (not deployer) and len(deployer_record.names): + target_deployer = deployer_record.names[0] deployment_request = { "record": { @@ -134,11 +217,14 @@ def command( "application": app, "version": "1.0.0", "name": f"{app_record.attributes.name}@{app_record.attributes.version}", - "deployer": deployer, + "deployer": target_deployer, "meta": {"when": str(datetime.utcnow())}, } } + if auction_id: + deployment_request["record"]["auction"] = auction_id + if config_ref: deployment_request["record"]["config"] = {"ref": config_ref} @@ -165,11 +251,12 @@ def command( elif use_payment: deployment_request["record"]["payment"] = use_payment + deployment_requests.append(deployment_request) + + # Send all requests + for deployment_request in deployment_requests: if dry_run: print(yaml.dump(deployment_request)) - return + continue - # Send the request laconic.publish(deployment_request) - finally: - shutil.rmtree(tempdir, ignore_errors=True) diff --git a/stack_orchestrator/deploy/webapp/request_webapp_undeployment.py b/stack_orchestrator/deploy/webapp/request_webapp_undeployment.py new file mode 100644 index 00000000..80cee3ce --- /dev/null +++ b/stack_orchestrator/deploy/webapp/request_webapp_undeployment.py @@ -0,0 +1,106 @@ +# Copyright ©2023 Vulcanize +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys + +import click +import yaml + +from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient) + + +def fatal(msg: str): + print(msg, file=sys.stderr) + sys.exit(1) + + +@click.command() +@click.option( + "--laconic-config", help="Provide a config file for laconicd", required=True +) +@click.option( + "--deployer", + help="The LRN of the deployer to process this request.", + required=True +) +@click.option( + "--deployment", + help="Deployment record (ApplicationDeploymentRecord) id of the deployment to remove.", + required=True, +) +@click.option( + "--make-payment", + help="The payment to make (in alnt). The value should be a number or 'auto' to use the deployer's minimum required payment.", +) +@click.option( + "--use-payment", help="The TX id of an existing, unused payment", default=None +) +@click.option( + "--dry-run", + help="Don't publish anything, just report what would be done.", + is_flag=True, +) +@click.pass_context +def command( + ctx, + laconic_config, + deployer, + deployment, + make_payment, + use_payment, + dry_run, +): + if make_payment and use_payment: + fatal("Cannot use --make-payment and --use-payment at the same time.") + + laconic = LaconicRegistryClient(laconic_config) + + deployer_record = laconic.get_record(deployer) + if not deployer_record: + fatal(f"Unable to locate deployer: {deployer}") + + undeployment_request = { + "record": { + "type": "ApplicationDeploymentRemovalRequest", + "version": "1.0.0", + "deployer": deployer, + "deployment": deployment, + } + } + + if make_payment: + amount = 0 + if dry_run: + undeployment_request["record"]["payment"] = "DRY_RUN" + elif "auto" == make_payment: + if "minimumPayment" in deployer_record.attributes: + amount = int( + deployer_record.attributes.minimumPayment.replace("alnt", "") + ) + else: + amount = make_payment + if amount: + receipt = laconic.send_tokens( + deployer_record.attributes.paymentAddress, amount + ) + undeployment_request["record"]["payment"] = receipt.tx.hash + print("Payment TX:", receipt.tx.hash) + elif use_payment: + undeployment_request["record"]["payment"] = use_payment + + if dry_run: + print(yaml.dump(undeployment_request)) + return + + laconic.publish(undeployment_request) diff --git a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py index cbab5896..b134e158 100644 --- a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py @@ -311,6 +311,7 @@ def command( # noqa: C901 main_logger.log(f"ERROR examining {r.id}: {e}") requests_to_execute = [] + # TODO: Handle requests with auction if min_required_payment: for r in requests_to_check_for_payment: main_logger.log(f"{r.id}: Confirming payment...") diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index c6b6a786..7b8914c0 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -24,6 +24,19 @@ import tempfile import uuid import yaml +from enum import Enum + + +class AuctionStatus(str, Enum): + COMMIT = "commit" + REVEAL = "reveal" + COMPLETED = "completed" + EXPIRED = "expired" + + +TOKEN_DENOM = "alnt" +AUCTION_KIND_PROVIDER = "provider" + class AttrDict(dict): def __init__(self, *args, **kwargs): @@ -58,6 +71,12 @@ class TimedLogger: self.last = datetime.datetime.now() +def load_known_requests(filename): + if filename and os.path.exists(filename): + return json.load(open(filename, "r")) + return {} + + def logged_cmd(log_file, *vargs): result = None try: @@ -92,74 +111,6 @@ def is_id(name_or_id: str): return not is_lrn(name_or_id) -def confirm_payment(laconic, record, payment_address, min_amount, logger): - req_owner = laconic.get_owner(record) - if req_owner == payment_address: - # No need to confirm payment if the sender and recipient are the same account. - return True - - if not record.attributes.payment: - logger.log(f"{record.id}: no payment tx info") - return False - - tx = laconic.get_tx(record.attributes.payment) - if not tx: - logger.log(f"{record.id}: cannot locate payment tx") - return False - - if tx.code != 0: - logger.log( - f"{record.id}: payment tx {tx.hash} was not successful - code: {tx.code}, log: {tx.log}" - ) - return False - - if tx.sender != req_owner: - logger.log( - f"{record.id}: payment sender {tx.sender} in tx {tx.hash} does not match deployment " - f"request owner {req_owner}" - ) - return False - - if tx.recipient != payment_address: - logger.log( - f"{record.id}: payment recipient {tx.recipient} in tx {tx.hash} does not match {payment_address}" - ) - return False - - pay_denom = "".join([i for i in tx.amount if not i.isdigit()]) - if pay_denom != "alnt": - logger.log( - f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected payment denomination" - ) - return False - - pay_amount = int("".join([i for i in tx.amount if i.isdigit()])) - if pay_amount < min_amount: - logger.log( - f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}" - ) - return False - - # Check if the payment was already used on a - used = laconic.app_deployments( - {"deployer": payment_address, "payment": tx.hash}, all=True - ) - if len(used): - logger.log(f"{record.id}: payment {tx.hash} already used on deployment {used}") - return False - - used = laconic.app_deployment_removals( - {"deployer": payment_address, "payment": tx.hash}, all=True - ) - if len(used): - logger.log( - f"{record.id}: payment {tx.hash} already used on deployment removal {used}" - ) - return False - - return True - - class LaconicRegistryClient: def __init__(self, config_file, log_file=None): self.config_file = config_file @@ -370,6 +321,34 @@ class LaconicRegistryClient: if require: raise Exception("Cannot locate tx:", hash) + def get_auction(self, auction_id, require=False): + args = [ + "laconic", + "-c", + self.config_file, + "registry", + "auction", + "get", + "--id", + auction_id, + ] + + results = None + try: + results = [ + AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r + ] + except: # noqa: E722 + pass + + if results and len(results): + return results[0] + + if require: + raise Exception("Cannot locate auction:", auction_id) + + return None + def app_deployment_requests(self, criteria=None, all=True): if criteria is None: criteria = {} @@ -398,6 +377,20 @@ class LaconicRegistryClient: criteria["type"] = "ApplicationDeploymentRemovalRecord" return self.list_records(criteria, all) + def webapp_deployers(self, criteria=None, all=True): + if criteria is None: + criteria = {} + criteria = criteria.copy() + criteria["type"] = "WebappDeployer" + return self.list_records(criteria, all) + + def app_deployment_auctions(self, criteria=None, all=True): + if criteria is None: + criteria = {} + criteria = criteria.copy() + criteria["type"] = "ApplicationDeploymentAuction" + return self.list_records(criteria, all) + def publish(self, record, names=None): if names is None: names = [] @@ -471,6 +464,88 @@ class LaconicRegistryClient: return AttrDict(json.loads(logged_cmd(self.log_file, *args))) + def create_auction(self, auction): + if auction["kind"] == AUCTION_KIND_PROVIDER: + args = [ + "laconic", + "-c", + self.config_file, + "registry", + "auction", + "create", + "--kind", + auction["kind"], + "--commits-duration", + str(auction["commits_duration"]), + "--reveals-duration", + str(auction["reveals_duration"]), + "--denom", + auction["denom"], + "--commit-fee", + str(auction["commit_fee"]), + "--reveal-fee", + str(auction["reveal_fee"]), + "--max-price", + str(auction["max_price"]), + "--num-providers", + str(auction["num_providers"]) + ] + else: + args = [ + "laconic", + "-c", + self.config_file, + "registry", + "auction", + "create", + "--kind", + auction["kind"], + "--commits-duration", + str(auction["commits_duration"]), + "--reveals-duration", + str(auction["reveals_duration"]), + "--denom", + auction["denom"], + "--commit-fee", + str(auction["commit_fee"]), + "--reveal-fee", + str(auction["reveal_fee"]), + "--minimum-bid", + str(auction["minimum_bid"]) + ] + + return json.loads(logged_cmd(self.log_file, *args))["auctionId"] + + def commit_bid(self, auction_id, amount, type="alnt"): + args = [ + "laconic", + "-c", + self.config_file, + "registry", + "auction", + "bid", + "commit", + auction_id, + str(amount), + type, + ] + + return json.loads(logged_cmd(self.log_file, *args))["reveal_file"] + + def reveal_bid(self, auction_id, reveal_file_path): + logged_cmd( + self.log_file, + "laconic", + "-c", + self.config_file, + "registry", + "auction", + "bid", + "reveal", + auction_id, + reveal_file_path, + ) + def file_hash(filename): return hashlib.sha1(open(filename).read().encode()).hexdigest() @@ -677,12 +752,15 @@ def publish_deployment( }, } } + if app_deployment_request: new_deployment_record["record"]["request"] = app_deployment_request.id - if app_deployment_request.attributes.payment: - new_deployment_record["record"][ - "payment" - ] = app_deployment_request.attributes.payment + + # Set auction or payment id from request + if app_deployment_request.attributes.auction: + new_deployment_record["record"]["auction"] = app_deployment_request.attributes.auction + elif app_deployment_request.attributes.payment: + new_deployment_record["record"]["payment"] = app_deployment_request.attributes.payment if webapp_deployer_record: new_deployment_record["record"]["deployer"] = webapp_deployer_record.names[0] @@ -730,3 +808,103 @@ def skip_by_tag(r, include_tags, exclude_tags): return True return False + + +def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min_amount, logger): + req_owner = laconic.get_owner(record) + if req_owner == payment_address: + # No need to confirm payment if the sender and recipient are the same account. + return True + + if not record.attributes.payment: + logger.log(f"{record.id}: no payment tx info") + return False + + tx = laconic.get_tx(record.attributes.payment) + if not tx: + logger.log(f"{record.id}: cannot locate payment tx") + return False + + if tx.code != 0: + logger.log( + f"{record.id}: payment tx {tx.hash} was not successful - code: {tx.code}, log: {tx.log}" + ) + return False + + if tx.sender != req_owner: + logger.log( + f"{record.id}: payment sender {tx.sender} in tx {tx.hash} does not match deployment " + f"request owner {req_owner}" + ) + return False + + if tx.recipient != payment_address: + logger.log( + f"{record.id}: payment recipient {tx.recipient} in tx {tx.hash} does not match {payment_address}" + ) + return False + + pay_denom = "".join([i for i in tx.amount if not i.isdigit()]) + if pay_denom != "alnt": + logger.log( + f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected payment denomination" + ) + return False + + pay_amount = int("".join([i for i in tx.amount if i.isdigit()])) + if pay_amount < min_amount: + logger.log( + f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}" + ) + return False + + # Check if the payment was already used on a + used = laconic.app_deployments( + {"deployer": payment_address, "payment": tx.hash}, all=True + ) + if len(used): + logger.log(f"{record.id}: payment {tx.hash} already used on deployment {used}") + return False + + used = laconic.app_deployment_removals( + {"deployer": payment_address, "payment": tx.hash}, all=True + ) + if len(used): + logger.log( + f"{record.id}: payment {tx.hash} already used on deployment removal {used}" + ) + return False + + return True + + +def confirm_auction(laconic: LaconicRegistryClient, record, deployer_lrn, payment_address, logger): + auction_id = record.attributes.auction + auction = laconic.get_auction(auction_id) + + # Fetch auction record for given auction + auction_records_by_id = laconic.app_deployment_auctions({"auction": auction_id}) + if len(auction_records_by_id) == 0: + logger.log(f"{record.id}: unable to locate record for auction {auction_id}") + return False + + # Cross check app against application in the auction record + requested_app = laconic.get_record(record.attributes.application, require=True) + auction_app = laconic.get_record(auction_records_by_id[0].attributes.application, require=True) + if requested_app.id != auction_app.id: + logger.log( + f"{record.id}: requested application {record.attributes.application} does not match application from " + f"auction record {auction_records_by_id[0].attributes.application}" + ) + return False + + if not auction: + logger.log(f"{record.id}: unable to locate auction {auction_id}") + return False + + # Check if the deployer payment address is in auction winners list + if payment_address not in auction.winnerAddresses: + logger.log(f"{record.id}: deployer payment address not in auction winners.") + return False + + return True diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index 5ae10468..a50c7c9b 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -26,7 +26,10 @@ from stack_orchestrator.deploy.webapp import (run_webapp, deploy_webapp_from_registry, undeploy_webapp_from_registry, publish_webapp_deployer, - request_webapp_deployment) + publish_deployment_auction, + handle_deployment_auction, + request_webapp_deployment, + request_webapp_undeployment) from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment @@ -64,7 +67,10 @@ cli.add_command(deploy_webapp.command, "deploy-webapp") cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry") cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry") cli.add_command(publish_webapp_deployer.command, "publish-deployer-to-registry") +cli.add_command(publish_deployment_auction.command, "publish-deployment-auction") +cli.add_command(handle_deployment_auction.command, "handle-deployment-auction") cli.add_command(request_webapp_deployment.command, "request-webapp-deployment") +cli.add_command(request_webapp_undeployment.command, "request-webapp-undeployment") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment")