Use file existence for registry mutex #959

Merged
ashwin merged 3 commits from deep-stack/stack-orchestrator:pm-nonblocking-mutex into main 2024-10-29 04:05:36 +00:00
2 changed files with 17 additions and 6 deletions
Showing only changes of commit 6caf131236 - Show all commits

View File

@ -1,4 +1,3 @@
import fcntl
from functools import wraps
import os
import time
@ -9,7 +8,11 @@ LOCK_TIMEOUT = 30
LOCK_RETRY_INTERVAL = 3
def acquire_lock(lock_file_path, timeout):
def acquire_lock(client, lock_file_path, timeout):
# Lock alreay acquired by the current client
if client.mutex_lock_acquired:
return
while True:
try:
# Check if lock file exists and is potentially stale
@ -30,6 +33,8 @@ def acquire_lock(lock_file_path, timeout):
fd = os.open(lock_file_path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
with os.fdopen(fd, 'w') as lock_file:
lock_file.write(str(time.time()))
client.mutex_lock_acquired = True
print(f"Registry lock acquired, {lock_file_path}")
# Lock successfully acquired
@ -39,14 +44,18 @@ def acquire_lock(lock_file_path, timeout):
print(f"Lock file {lock_file_path} exists, waiting...")
time.sleep(LOCK_RETRY_INTERVAL)
def release_lock(lock_file_path):
def release_lock(client, lock_file_path):
try:
os.remove(lock_file_path)
client.mutex_lock_acquired = False
print(f"Registry lock released, {lock_file_path}")
except FileNotFoundError:
# Lock file already removed
pass
def registry_mutex():
def decorator(func):
@wraps(func)
@ -56,12 +65,12 @@ def registry_mutex():
lock_file_path = self.mutex_lock_file
# Acquire the lock before running the function
acquire_lock(lock_file_path, LOCK_TIMEOUT)
acquire_lock(self, lock_file_path, LOCK_TIMEOUT)
try:
return func(self, *args, **kwargs)
finally:
# Release the lock after the function completes
release_lock(lock_file_path)
release_lock(self, lock_file_path)
return wrapper

View File

@ -117,7 +117,6 @@ class LaconicRegistryClient:
def __init__(self, config_file, log_file=None, mutex_lock_file=None):
self.config_file = config_file
self.log_file = log_file
self.mutex_lock_file = mutex_lock_file
self.cache = AttrDict(
{
"name_or_id": {},
@ -126,6 +125,9 @@ class LaconicRegistryClient:
}
)
self.mutex_lock_file = mutex_lock_file
self.mutex_lock_acquired = False
def whoami(self, refresh=False):
if not refresh and "whoami" in self.cache:
return self.cache["whoami"]