Compare commits

..

5 Commits

Author SHA1 Message Date
39df4683ac Allow payment reuse for same app LRN (#961)
All checks were successful
Lint Checks / Run linter (push) Successful in 33s
Publish / Build and publish (push) Successful in 1m12s
Deploy Test / Run deploy test suite (push) Successful in 4m54s
Smoke Test / Run basic test suite (push) Successful in 3m52s
Webapp Test / Run webapp test suite (push) Successful in 4m38s
Fixturenet-Laconicd-Test / Run Laconicd fixturenet and Laconic CLI tests (push) Successful in 19m17s
K8s Deploy Test / Run deploy test suite on kind/k8s (push) Successful in 7m48s
Database Test / Run database hosting test on kind/k8s (push) Successful in 9m27s
Container Registry Test / Run contaier registry hosting test on kind/k8s (push) Successful in 3m35s
External Stack Test / Run external stack test suite (push) Successful in 4m35s
Part of [Service provider auctions for web deployments](https://www.notion.so/Service-provider-auctions-for-web-deployments-104a6b22d47280dbad51d28aa3a91d75)

Reviewed-on: #961
Reviewed-by: ashwin <ashwin@noreply.git.vdb.to>
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
2024-10-29 11:30:03 +00:00
23ca4c4341 Allow payment reuse for application redeployment (#960)
All checks were successful
Lint Checks / Run linter (push) Successful in 39s
Publish / Build and publish (push) Successful in 1m10s
Smoke Test / Run basic test suite (push) Successful in 3m54s
Webapp Test / Run webapp test suite (push) Successful in 4m40s
Deploy Test / Run deploy test suite (push) Successful in 4m51s
Part of [Service provider auctions for web deployments](https://www.notion.so/Service-provider-auctions-for-web-deployments-104a6b22d47280dbad51d28aa3a91d75)

Reviewed-on: #960
Reviewed-by: ashwin <ashwin@noreply.git.vdb.to>
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
2024-10-29 06:51:48 +00:00
f64ef5d128 Use file existence for registry mutex (#959)
All checks were successful
Lint Checks / Run linter (push) Successful in 1m1s
Publish / Build and publish (push) Successful in 1m27s
Webapp Test / Run webapp test suite (push) Successful in 4m59s
Smoke Test / Run basic test suite (push) Successful in 4m10s
Deploy Test / Run deploy test suite (push) Successful in 5m33s
Part of [Service provider auctions for web deployments](https://www.notion.so/Service-provider-auctions-for-web-deployments-104a6b22d47280dbad51d28aa3a91d75)

Reviewed-on: #959
Reviewed-by: ashwin <ashwin@noreply.git.vdb.to>
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
2024-10-29 04:05:35 +00:00
5f8e809b2d Add mutex lock file path to registry CLI wrapper class (#958)
All checks were successful
Lint Checks / Run linter (push) Successful in 33s
Publish / Build and publish (push) Successful in 1m24s
Deploy Test / Run deploy test suite (push) Successful in 4m53s
Webapp Test / Run webapp test suite (push) Successful in 4m39s
Smoke Test / Run basic test suite (push) Successful in 3m58s
Fixturenet-Laconicd-Test / Run Laconicd fixturenet and Laconic CLI tests (push) Successful in 19m38s
K8s Deploy Test / Run deploy test suite on kind/k8s (push) Successful in 6m59s
Database Test / Run database hosting test on kind/k8s (push) Successful in 9m59s
Container Registry Test / Run contaier registry hosting test on kind/k8s (push) Successful in 3m49s
External Stack Test / Run external stack test suite (push) Successful in 4m38s
Part of [Service provider auctions for web deployments](https://www.notion.so/Service-provider-auctions-for-web-deployments-104a6b22d47280dbad51d28aa3a91d75)
Follows #957

Reviewed-on: #958
Reviewed-by: ashwin <ashwin@noreply.git.vdb.to>
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
2024-10-28 06:03:13 +00:00
4a7df2de33 Use a mutex for registry CLI txs in webapp deployment commands (#957)
All checks were successful
Lint Checks / Run linter (push) Successful in 37s
Publish / Build and publish (push) Successful in 1m19s
Webapp Test / Run webapp test suite (push) Successful in 4m45s
Smoke Test / Run basic test suite (push) Successful in 4m16s
Deploy Test / Run deploy test suite (push) Successful in 4m58s
Fixturenet-Laconicd-Test / Run Laconicd fixturenet and Laconic CLI tests (push) Successful in 19m17s
K8s Deploy Test / Run deploy test suite on kind/k8s (push) Successful in 7m33s
Database Test / Run database hosting test on kind/k8s (push) Successful in 9m41s
Container Registry Test / Run contaier registry hosting test on kind/k8s (push) Successful in 3m36s
External Stack Test / Run external stack test suite (push) Successful in 4m43s
Part of [Service provider auctions for web deployments](https://www.notion.so/Service-provider-auctions-for-web-deployments-104a6b22d47280dbad51d28aa3a91d75) and #948

- Add a registry mutex decorator over tx methods in `LaconicRegistryClient` wrapper
- Required to allow multiple process to run webapp deployment tooling without running into account sequence errors when sending laconicd txs

Reviewed-on: #957
Reviewed-by: ashwin <ashwin@noreply.git.vdb.to>
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
2024-10-25 08:40:54 +00:00
6 changed files with 142 additions and 60 deletions

View File

@ -360,6 +360,9 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option( @click.option(
"--private-key-file", help="The private key for decrypting config.", required=True "--private-key-file", help="The private key for decrypting config.", required=True
) )
@click.option(
"--registry-lock-file", help="File path to use for registry mutex lock", default=None
)
@click.option( @click.option(
"--private-key-passphrase", "--private-key-passphrase",
help="The passphrase for the private key.", help="The passphrase for the private key.",
@ -393,6 +396,7 @@ def command( # noqa: C901
private_key_passphrase, private_key_passphrase,
all_requests, all_requests,
auction_requests, auction_requests,
registry_lock_file,
): ):
if request_id and discover: if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr) print("Cannot specify both --request-id and --discover", file=sys.stderr)
@ -444,7 +448,7 @@ def command( # noqa: C901
include_tags = [tag.strip() for tag in include_tags.split(",") if tag] include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag] exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr) laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file)
webapp_deployer_record = laconic.get_record(lrn, require=True) webapp_deployer_record = laconic.get_record(lrn, require=True)
payment_address = webapp_deployer_record.attributes.paymentAddress payment_address = webapp_deployer_record.attributes.paymentAddress
main_logger.log(f"Payment address: {payment_address}") main_logger.log(f"Payment address: {payment_address}")
@ -649,7 +653,7 @@ def command( # noqa: C901
) )
run_log_file = open(run_log_file_path, "wt") run_log_file = open(run_log_file_path, "wt")
run_reg_client = LaconicRegistryClient( run_reg_client = LaconicRegistryClient(
laconic_config, log_file=run_log_file laconic_config, log_file=run_log_file, mutex_lock_file=registry_lock_file
) )
build_logger = TimedLogger(run_id, run_log_file) build_logger = TimedLogger(run_id, run_log_file)

View File

@ -120,6 +120,9 @@ def dump_known_auction_requests(filename, requests, status="SEEN"):
help="Bid to place on application deployment auctions (in alnt)", help="Bid to place on application deployment auctions (in alnt)",
required=True, required=True,
) )
@click.option(
"--registry-lock-file", help="File path to use for registry mutex lock", default=None
)
@click.option( @click.option(
"--dry-run", help="Don't do anything, just report what would be done.", is_flag=True "--dry-run", help="Don't do anything, just report what would be done.", is_flag=True
) )
@ -129,6 +132,7 @@ def command(
laconic_config, laconic_config,
state_file, state_file,
bid_amount, bid_amount,
registry_lock_file,
dry_run, dry_run,
): ):
if int(bid_amount) < 0: if int(bid_amount) < 0:
@ -138,7 +142,7 @@ def command(
logger = TimedLogger(file=sys.stderr) logger = TimedLogger(file=sys.stderr)
try: try:
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr) laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file)
auctions_requests = laconic.app_deployment_auctions() auctions_requests = laconic.app_deployment_auctions()
previous_requests = {} previous_requests = {}

View File

@ -102,7 +102,7 @@ def command(
"max_price": max_price, "max_price": max_price,
"num_providers": num_providers, "num_providers": num_providers,
} }
auction_id = laconic.create_auction(provider_auction_params) auction_id = laconic.create_deployment_auction(provider_auction_params)
print("Deployment auction created:", auction_id) print("Deployment auction created:", auction_id)
if not auction_id: if not auction_id:

View File

@ -0,0 +1,77 @@
from functools import wraps
import os
import time
# Define default file path for the lock
DEFAULT_LOCK_FILE_PATH = "/tmp/registry_mutex_lock_file"
LOCK_TIMEOUT = 30
LOCK_RETRY_INTERVAL = 3
def acquire_lock(client, lock_file_path, timeout):
# Lock alreay acquired by the current client
if client.mutex_lock_acquired:
return
while True:
try:
# Check if lock file exists and is potentially stale
if os.path.exists(lock_file_path):
with open(lock_file_path, 'r') as lock_file:
timestamp = float(lock_file.read().strip())
# If lock is stale, remove the lock file
if time.time() - timestamp > timeout:
print(f"Stale lock detected, removing lock file {lock_file_path}")
os.remove(lock_file_path)
else:
print(f"Lock file {lock_file_path} exists and is recent, waiting...")
time.sleep(LOCK_RETRY_INTERVAL)
continue
# Try to create a new lock file with the current timestamp
fd = os.open(lock_file_path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
with os.fdopen(fd, 'w') as lock_file:
lock_file.write(str(time.time()))
client.mutex_lock_acquired = True
print(f"Registry lock acquired, {lock_file_path}")
# Lock successfully acquired
return
except FileExistsError:
print(f"Lock file {lock_file_path} exists, waiting...")
time.sleep(LOCK_RETRY_INTERVAL)
def release_lock(client, lock_file_path):
try:
os.remove(lock_file_path)
client.mutex_lock_acquired = False
print(f"Registry lock released, {lock_file_path}")
except FileNotFoundError:
# Lock file already removed
pass
def registry_mutex():
def decorator(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
lock_file_path = DEFAULT_LOCK_FILE_PATH
if self.mutex_lock_file:
lock_file_path = self.mutex_lock_file
# Acquire the lock before running the function
acquire_lock(self, lock_file_path, LOCK_TIMEOUT)
try:
return func(self, *args, **kwargs)
finally:
# Release the lock after the function completes
release_lock(self, lock_file_path)
return wrapper
return decorator

View File

@ -178,6 +178,9 @@ def dump_known_requests(filename, requests):
"my payment address are examined).", "my payment address are examined).",
is_flag=True, is_flag=True,
) )
@click.option(
"--registry-lock-file", help="File path to use for registry mutex lock", default=None
)
@click.pass_context @click.pass_context
def command( # noqa: C901 def command( # noqa: C901
ctx, ctx,
@ -195,6 +198,7 @@ def command( # noqa: C901
min_required_payment, min_required_payment,
lrn, lrn,
all_requests, all_requests,
registry_lock_file,
): ):
if request_id and discover: if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr) print("Cannot specify both --request-id and --discover", file=sys.stderr)
@ -212,7 +216,7 @@ def command( # noqa: C901
include_tags = [tag.strip() for tag in include_tags.split(",") if tag] include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag] exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr) laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file)
deployer_record = laconic.get_record(lrn, require=True) deployer_record = laconic.get_record(lrn, require=True)
payment_address = deployer_record.attributes.paymentAddress payment_address = deployer_record.attributes.paymentAddress
main_logger.log(f"Payment address: {payment_address}") main_logger.log(f"Payment address: {payment_address}")

View File

@ -26,6 +26,8 @@ import yaml
from enum import Enum from enum import Enum
from stack_orchestrator.deploy.webapp.registry_mutex import registry_mutex
class AuctionStatus(str, Enum): class AuctionStatus(str, Enum):
COMMIT = "commit" COMMIT = "commit"
@ -112,7 +114,7 @@ def is_id(name_or_id: str):
class LaconicRegistryClient: class LaconicRegistryClient:
def __init__(self, config_file, log_file=None): def __init__(self, config_file, log_file=None, mutex_lock_file=None):
self.config_file = config_file self.config_file = config_file
self.log_file = log_file self.log_file = log_file
self.cache = AttrDict( self.cache = AttrDict(
@ -123,6 +125,9 @@ class LaconicRegistryClient:
} }
) )
self.mutex_lock_file = mutex_lock_file
self.mutex_lock_acquired = False
def whoami(self, refresh=False): def whoami(self, refresh=False):
if not refresh and "whoami" in self.cache: if not refresh and "whoami" in self.cache:
return self.cache["whoami"] return self.cache["whoami"]
@ -391,6 +396,7 @@ class LaconicRegistryClient:
criteria["type"] = "ApplicationDeploymentAuction" criteria["type"] = "ApplicationDeploymentAuction"
return self.list_records(criteria, all) return self.list_records(criteria, all)
@registry_mutex()
def publish(self, record, names=None): def publish(self, record, names=None):
if names is None: if names is None:
names = [] names = []
@ -421,6 +427,7 @@ class LaconicRegistryClient:
finally: finally:
logged_cmd(self.log_file, "rm", "-rf", tmpdir) logged_cmd(self.log_file, "rm", "-rf", tmpdir)
@registry_mutex()
def set_name(self, name, record_id): def set_name(self, name, record_id):
logged_cmd( logged_cmd(
self.log_file, self.log_file,
@ -434,6 +441,7 @@ class LaconicRegistryClient:
record_id, record_id,
) )
@registry_mutex()
def delete_name(self, name): def delete_name(self, name):
logged_cmd( logged_cmd(
self.log_file, self.log_file,
@ -446,6 +454,7 @@ class LaconicRegistryClient:
name, name,
) )
@registry_mutex()
def send_tokens(self, address, amount, type="alnt"): def send_tokens(self, address, amount, type="alnt"):
args = [ args = [
"laconic", "laconic",
@ -464,8 +473,8 @@ class LaconicRegistryClient:
return AttrDict(json.loads(logged_cmd(self.log_file, *args))) return AttrDict(json.loads(logged_cmd(self.log_file, *args)))
def create_auction(self, auction): @registry_mutex()
if auction["kind"] == AUCTION_KIND_PROVIDER: def create_deployment_auction(self, auction):
args = [ args = [
"laconic", "laconic",
"-c", "-c",
@ -490,32 +499,10 @@ class LaconicRegistryClient:
"--num-providers", "--num-providers",
str(auction["num_providers"]) str(auction["num_providers"])
] ]
else:
args = [
"laconic",
"-c",
self.config_file,
"registry",
"auction",
"create",
"--kind",
auction["kind"],
"--commits-duration",
str(auction["commits_duration"]),
"--reveals-duration",
str(auction["reveals_duration"]),
"--denom",
auction["denom"],
"--commit-fee",
str(auction["commit_fee"]),
"--reveal-fee",
str(auction["reveal_fee"]),
"--minimum-bid",
str(auction["minimum_bid"])
]
return json.loads(logged_cmd(self.log_file, *args))["auctionId"] return json.loads(logged_cmd(self.log_file, *args))["auctionId"]
@registry_mutex()
def commit_bid(self, auction_id, amount, type="alnt"): def commit_bid(self, auction_id, amount, type="alnt"):
args = [ args = [
"laconic", "laconic",
@ -532,6 +519,7 @@ class LaconicRegistryClient:
return json.loads(logged_cmd(self.log_file, *args))["reveal_file"] return json.loads(logged_cmd(self.log_file, *args))["reveal_file"]
@registry_mutex()
def reveal_bid(self, auction_id, reveal_file_path): def reveal_bid(self, auction_id, reveal_file_path):
logged_cmd( logged_cmd(
self.log_file, self.log_file,
@ -858,16 +846,21 @@ def confirm_payment(laconic: LaconicRegistryClient, record, payment_address, min
) )
return False return False
# Check if the payment was already used on a # Check if the payment was already used on a deployment
used = laconic.app_deployments( used = laconic.app_deployments(
{"deployer": payment_address, "payment": tx.hash}, all=True {"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
) )
if len(used): if len(used):
logger.log(f"{record.id}: payment {tx.hash} already used on deployment {used}") # Fetch the app name from request record
used_request = laconic.get_record(used[0].attributes.request, require=True)
# Check that payment was used for deployment of same application
if record.attributes.application != used_request.attributes.application:
logger.log(f"{record.id}: payment {tx.hash} already used on a different application deployment {used}")
return False return False
used = laconic.app_deployment_removals( used = laconic.app_deployment_removals(
{"deployer": payment_address, "payment": tx.hash}, all=True {"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
) )
if len(used): if len(used):
logger.log( logger.log(