Use non-blocking mutex on registry CLI with retry on error

stack_orchestrator/deploy/webapp/registry_mutex.py

@@ -1,9 +1,12 @@
 import fcntl
 from functools import wraps
+import time
 
 # Define default file path for the lock
 DEFAULT_LOCK_FILE_PATH = "/tmp/registry_mutex_lock_file"
 
+LOCK_RETRY_INTERVAL = 3
+
 
 def registry_mutex():
     def decorator(func):
@@ -13,16 +16,25 @@ def registry_mutex():
             if self.mutex_lock_file:
                 lock_file_path = self.mutex_lock_file
 
+            result = None
             with open(lock_file_path, 'w') as lock_file:
-                try:
-                    # Try to acquire the lock
-                    fcntl.flock(lock_file, fcntl.LOCK_EX)
+                while True:
+                    try:
+                        # Try to acquire the lock
+                        fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                        print(f"Registry lock acquired, {lock_file_path}")
 
-                    # Call the actual function
-                    result = func(self, *args, **kwargs)
-                finally:
-                    # Always release the lock
-                    fcntl.flock(lock_file, fcntl.LOCK_UN)
+                        # Call the actual function
+                        result = func(self, *args, **kwargs)
+                        break
+                    except BlockingIOError:
+                        # Retry on error
+                        print(f"Not able to acquire lock on {lock_file_path}, retrying in {LOCK_RETRY_INTERVAL}s...")
+                        time.sleep(LOCK_RETRY_INTERVAL)
+
+                # Always release the lock
+                fcntl.flock(lock_file, fcntl.LOCK_UN)
+                print(f"Registry lock released, {lock_file_path}")
 
             return result