Fix: Worker_settings overwrite & Watchdog Celery, Scripts

Added watchdog for celery in dev mode to apply changes
Fix Overwritten of worker settings. Priority keys over .env overwrites
Chore: Script fixes of master,worker

Dockerfile

@@ -25,5 +25,7 @@ RUN pip install --upgrade pip \
 # Copy project files
 COPY . .
 
+RUN chmod +x /app/app/entrypoint.sh
+
 # Default entrypoint (can be overridden in docker-compose)
 CMD ["bash"]

app/Dockerfile

@@ -25,5 +25,7 @@ RUN pip install --upgrade pip \
 # Copy project files
 COPY . .
 
+RUN chmod +x /app/entrypoint.sh
+
 # Default entrypoint (can be overridden in docker-compose)
 CMD ["bash"]

app/entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+# Wraps any command with watchmedo auto-restart in development mode.
+# In production, just exec the command directly.
+if [ "${APP_ENV:-development}" = "development" ]; then
+    exec watchmedo auto-restart \
+        --directory=. \
+        --pattern="*.py" \
+        --recursive \
+        -- "$@"
+else
+    exec "$@"
+fi

app/requirements.txt

@@ -18,4 +18,5 @@ apiflask
 marshmallow
 marshmallow_sqlalchemy
 cryptography
-flasgger
+flasgger
+watchdog

docker-compose.yml

@@ -89,7 +89,8 @@ services:
     build:
       context: .
     container_name: websocket-server
-    command: python3 -m websocket_server.main
+    entrypoint: ["/app/app/entrypoint.sh"]
+    command: ["python3", "-m", "websocket_server.main"]
     ports:
       - "6001:6001"
     env_file: .env
@@ -107,7 +108,8 @@ services:
     build:
       context: .
     container_name: vnc-proxy
-    command: python3 vnc_proxy.py
+    entrypoint: ["/app/app/entrypoint.sh"]
+    command: ["python3", "vnc_proxy.py"]
     ports:
       - "6002:6002"
     env_file: .env
@@ -141,7 +143,8 @@ services:
     build:
       context: app/
     container_name: celery-worker
-    command: celery -A app.celery_app.celery worker --loglevel=info
+    entrypoint: ["/app/app/entrypoint.sh"]
+    command: ["celery", "-A", "app.celery_app.celery", "worker", "--loglevel=info"]
     env_file: .env
     volumes:
       - .:/app
@@ -156,7 +159,8 @@ services:
     build:
       context: app/
     container_name: celery-beat
-    command: celery -A app.celery_app.celery beat --loglevel=info
+    entrypoint: ["/app/app/entrypoint.sh"]
+    command: ["celery", "-A", "app.celery_app.celery", "beat", "--loglevel=info"]
     env_file: .env
     volumes:
       - .:/app

requirements.txt

@@ -16,4 +16,5 @@ PyJWT
 python-dotenv
 psutil
 celery
-cryptography
+cryptography
+watchdog

run-master.sh

@@ -22,39 +22,34 @@ else
     exit 1
 fi
 
-case "$1" in
-    --help)
-        show_help
-        ;;
+CLEAN=false
+BUILD=false
 
-    --build)
-        echo "Building containers without cache..."
-        $DOCKER_COMPOSE down -v
-        $DOCKER_COMPOSE build --no-cache
-        echo "Starting services..."
-        $DOCKER_COMPOSE up -d
-        ;;
+for arg in "$@"; do
+    case "$arg" in
+        --clean) CLEAN=true ;;
+        --build) BUILD=true ;;
+        --help)  show_help; exit 0 ;;
+        *)
+            echo "Unknown option: $arg"
+            echo ""
+            show_help
+            exit 1
+            ;;
+    esac
+done
 
-    --clean)
-        echo "Cleaning containers, volumes, images, and orphans..."
-        $DOCKER_COMPOSE down -v --rmi all --remove-orphans
+if $CLEAN; then
+    echo "Cleaning containers, volumes, images, and orphans..."
+    $DOCKER_COMPOSE down -v --rmi all --remove-orphans
+    echo "Rebuilding containers without cache..."
+    $DOCKER_COMPOSE build --no-cache
+elif $BUILD; then
+    echo "Stopping existing containers before rebuild..."
+    $DOCKER_COMPOSE down -v
+    echo "Building containers without cache..."
+    $DOCKER_COMPOSE build --no-cache
+fi
 
-        echo "Rebuilding containers without cache..."
-        $DOCKER_COMPOSE build --no-cache
-
-        echo "Starting services..."
-        $DOCKER_COMPOSE up -d
-        ;;
-
-    "")
-        echo "Starting services..."
-        $DOCKER_COMPOSE up -d
-        ;;
-
-    *)
-        echo "Unknown option: $1"
-        echo ""
-        show_help
-        exit 1
-        ;;
-esac
+echo "Starting services..."
+$DOCKER_COMPOSE up -d

worker/docker-compose.yml

@@ -8,6 +8,11 @@ services:
     restart: unless-stopped
     env_file: .env
     volumes:
+      # The whole worker dir is bind-mounted, so worker_settings.json already
+      # persists on the host inside it. Do NOT add a single-file bind mount for
+      # worker_settings.json: when the host file is missing Docker creates it as a
+      # directory, which the worker can't read/write, so it loses its identity and
+      # re-enrolls as a new worker on every (re)start.
       - .:/app
       - /tmp/xcloudify:/tmp/xcloudify
       - /run/libvirt:/run/libvirt
@@ -15,7 +20,6 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - /var/run/openvswitch:/var/run/openvswitch
       - /run/openvswitch:/run/openvswitch
-      - ./worker_settings.json:/app/worker_settings.json
       - /run/netns:/run/netns
       - /var/run/docker/netns:/var/run/docker/netns
       - /proc:/proc:ro

worker/run-worker.sh

@@ -2,9 +2,13 @@
 
 set -e
 
-# Ensure worker_settings.json exists
+# Ensure worker_settings.json is a file (Docker can create it as a directory
+# on first bind-mount if the file doesn't exist on the host yet)
+if [ -d worker_settings.json ]; then
+    rm -rf worker_settings.json
+fi
 if [ ! -f worker_settings.json ]; then
-    touch worker_settings.json
+    echo '{}' > worker_settings.json
     echo "Created worker_settings.json"
 fi
 
@@ -81,7 +85,6 @@ case "$COMMAND" in
         $COMPOSE_CMD down -v --rmi all --remove-orphans
 
         echo "Rebuilding containers without cache..."
-        $COMPOSE_CMD down -v
         $COMPOSE_CMD build --no-cache
 
         echo "Starting services..."

worker/settings.py

@@ -19,6 +19,10 @@ class SettingsManager:
         # Add other essential settings keys here
     ]
 
+    # Identity keys that, once persisted, must survive restarts untouched. The
+    # file value always wins over .env/defaults so the worker is never re-enrolled
+    STICKY_SETTINGS = {"WORKER_ID", "WORKER_SECRET"}
+
     # Define settings that can optionally come from CLI args, mapping arg name to setting key
     CLI_OVERRIDABLE_SETTINGS = {
         "WEBSOCKET_SERVER_URL": "WEBSOCKET_SERVER_URL",
@@ -67,11 +71,33 @@ class SettingsManager:
             loaded_from_file = self._load_from_file()
             loaded_from_env = self._load_from_env()
 
-            # Merge settings: .env overrides file, file overrides defaults
-            merged_settings = {}
-            merged_settings.update(self.DEFAULT_SETTINGS) # Start with defaults
-            merged_settings.update(loaded_from_file)      # Layer file settings
-            merged_settings.update(loaded_from_env)      # Layer .env settings
+            # Per-field merge of persisted file settings and .env settings.
+            #
+            #   - Start from the built-in defaults.
+            #   - STICKY settings (worker identity: WORKER_ID / WORKER_SECRET) persist
+            #   - Every other key: the file value is kept when .env has nothing for it
+            #     (file preferred over default), and .env wins when it does provide a
+            #     value (operator config / 12-factor override).
+            #
+            # `_load_from_env` already drops empty/None values, so "present in env"
+            # means an actual, non-empty value.
+            merged_settings = dict(self.DEFAULT_SETTINGS)
+
+            for key in set(loaded_from_file) | set(loaded_from_env):
+                in_file = key in loaded_from_file
+                in_env = key in loaded_from_env
+                if key in self.STICKY_SETTINGS:
+                    # File always wins for sticky identity keys.
+                    if in_file:
+                        merged_settings[key] = loaded_from_file[key]
+                    elif in_env:
+                        merged_settings[key] = loaded_from_env[key]
+                else:
+                    # Env wins when present, otherwise fall back to the file value.
+                    if in_env:
+                        merged_settings[key] = loaded_from_env[key]
+                    elif in_file:
+                        merged_settings[key] = loaded_from_file[key]
 
             self._settings = merged_settings
             logger.info(f"Initial settings loaded and merged.")
@@ -82,6 +108,11 @@ class SettingsManager:
     def _load_from_file(self):
         """Loads settings from the JSON file."""
         try:
+            if os.path.isdir(self._settings_file):
+                import shutil
+                shutil.rmtree(self._settings_file)
+                logger.warning(f"Settings path {self._settings_file} was a directory — removed it. Starting fresh.")
+                return {}
             if os.path.exists(self._settings_file):
                 with open(self._settings_file, 'r') as f:
                     loaded_settings = json.load(f)
@@ -108,7 +139,9 @@ class SettingsManager:
             loaded_count = 0
             for key in all_known_keys:
                 value = get_worker_env(key)
-                if value is not None:
+                # Treat unset and empty/whitespace-only env vars as "not provided" so a
+                # blank line in .env (e.g. `WORKER_ID=`) can never clobber a persisted value.
+                if value is not None and value.strip() != "":
                     # Basic type conversion for known defaults
                     if key in self.DEFAULT_SETTINGS:
                         default_val = self.DEFAULT_SETTINGS[key]