From 59e12f0ab170ce832a17bedd611badd4b48013de Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Mon, 28 Oct 2024 14:32:19 +0530
Subject: [PATCH] Use non-blocking mutex on registry CLI with retry on error

---
 .../deploy/webapp/registry_mutex.py           | 28 +++++++++++++------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/stack_orchestrator/deploy/webapp/registry_mutex.py b/stack_orchestrator/deploy/webapp/registry_mutex.py
index 7c835f19..5e8e0219 100644
--- a/stack_orchestrator/deploy/webapp/registry_mutex.py
+++ b/stack_orchestrator/deploy/webapp/registry_mutex.py
@@ -1,9 +1,12 @@
 import fcntl
 from functools import wraps
+import time
 
 # Define default file path for the lock
 DEFAULT_LOCK_FILE_PATH = "/tmp/registry_mutex_lock_file"
 
+LOCK_RETRY_INTERVAL = 3
+
 
 def registry_mutex():
     def decorator(func):
@@ -13,16 +16,25 @@ def registry_mutex():
             if self.mutex_lock_file:
                 lock_file_path = self.mutex_lock_file
 
+            result = None
             with open(lock_file_path, 'w') as lock_file:
-                try:
-                    # Try to acquire the lock
-                    fcntl.flock(lock_file, fcntl.LOCK_EX)
+                while True:
+                    try:
+                        # Try to acquire the lock
+                        fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                        print(f"Registry lock acquired, {lock_file_path}")
 
-                    # Call the actual function
-                    result = func(self, *args, **kwargs)
-                finally:
-                    # Always release the lock
-                    fcntl.flock(lock_file, fcntl.LOCK_UN)
+                        # Call the actual function
+                        result = func(self, *args, **kwargs)
+                        break
+                    except BlockingIOError:
+                        # Retry on error
+                        print(f"Not able to acquire lock on {lock_file_path}, retrying in {LOCK_RETRY_INTERVAL}s...")
+                        time.sleep(LOCK_RETRY_INTERVAL)
+
+                # Always release the lock
+                fcntl.flock(lock_file, fcntl.LOCK_UN)
+                print(f"Registry lock released, {lock_file_path}")
 
             return result