Merge branch 'worker-async' into enrollment-key

.gitignore

@@ -3,4 +3,5 @@ instance/*
 logs/*
 .venv/
 mariadb_data/
-.env
+.env
+.aider*

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "app/static/novnc"]
+	path = app/static/novnc
+	url = https://github.com/novnc/noVNC.git

api_client/client.py

@@ -1,11 +1,9 @@
-# import streamlit as st
 import requests
-# This API client is used by streamlit and by the websocket server
 
 class IaaSClient:
     def __init__(self, api_key, base_url="http://localhost:5000/api"):
         self.base_url = base_url
-        self.api_key=api_key
+        self.api_key = api_key
         
     def _make_request(self, method, endpoint, data=None):
         headers = {
@@ -24,80 +22,245 @@ class IaaSClient:
             print(f"API Error: {str(e)}")
             return None
 
+    # Universe CRUD
     def get_universes(self):
         return self._make_request('GET', 'universes')
 
+    def get_universe(self, universe_id):
+        return self._make_request('GET', f'universes/{universe_id}')
+
     def create_universe(self, name, description):
         return self._make_request('POST', 'universes', {'name': name, 'description': description})
 
+    def update_universe(self, universe_id, data):
+        return self._make_request('PUT', f'universes/{universe_id}', data)
+
+    def delete_universe(self, universe_id):
+        return self._make_request('DELETE', f'universes/{universe_id}')
+
+    # Project CRUD
     def get_projects(self):
         return self._make_request('GET', 'projects')
 
+    def get_project(self, project_id):
+        return self._make_request('GET', f'projects/{project_id}')
+
     def create_project(self, name, universe_id):
         return self._make_request('POST', 'projects', {'name': name, 'universe_id': universe_id})
 
-    def get_vdcs(self):
-        return self._make_request('GET', 'vdcs')
+    def update_project(self, project_id, data):
+        return self._make_request('PUT', f'projects/{project_id}', data)
 
-    def create_vdc(self, name, project_id):
-        return self._make_request('POST', 'vdcs', {'name': name, 'project_id': project_id})
-    
+    def delete_project(self, project_id):
+        return self._make_request('DELETE', f'projects/{project_id}')
+
+    # virtual_data_centers CRUD
+    def get_virtual_data_centers(self):
+        return self._make_request('GET', 'virtual_data_centers')
+
+    def get_virtual_data_center(self, virtual_data_center_id):
+        return self._make_request('GET', f'virtual_data_centers/{virtual_data_center_id}')
+
+    def create_virtual_data_center(self, name, project_id):
+        return self._make_request('POST', 'virtual_data_centers', {'name': name, 'project_id': project_id})
+
+    def update_virtual_data_center(self, virtual_data_center_id, data):
+        return self._make_request('PUT', f'virtual_data_centers/{virtual_data_center_id}', data)
+
+    def delete_virtual_data_center(self, virtual_data_center_id):
+        return self._make_request('DELETE', f'virtual_data_centers/{virtual_data_center_id}')
+
+    # Network CRUD
     def get_networks(self):
         return self._make_request('GET', 'networks')
 
-    def create_network(self, name, vdc_id):
-        return self._make_request('POST', 'networks', {'name': name, 'vdc_id': vdc_id})
+    def get_network(self, network_id):
+        return self._make_request('GET', f'networks/{network_id}')
+
+    def create_network(self, name, virtual_data_center_id):
+        return self._make_request('POST', 'networks', {'name': name, 'virtual_data_center_id': virtual_data_center_id})
+
+    def update_network(self, network_id, data):
+        return self._make_request('PUT', f'networks/{network_id}', data)
+
+    def delete_network(self, network_id):
+        return self._make_request('DELETE', f'networks/{network_id}')
+
+    # Network Port CRUD
+    def get_network_ports(self):
+        return self._make_request('GET', 'network_ports')
+
+    def get_network_port(self, network_port_id):
+        return self._make_request('GET', f'network_ports/{network_port_id}')
+
+    def create_network_port(self, data):
+        return self._make_request('POST', 'network_ports', data)
+
+    def update_network_port(self, network_port_id, data):
+        return self._make_request('PUT', f'network_ports/{network_port_id}', data)
+
+    def delete_network_port(self, network_port_id):
+        return self._make_request('DELETE', f'network_ports/{network_port_id}')
     
     def get_network_ports_by_network(self,network_id):
         return self._make_request('GET', f'network_ports/by_network/{network_id}')
         
+
+    # Region CRUD
     def get_regions(self):
         return self._make_request('GET', 'regions')
-    
+
+    def get_region(self, region_id):
+        return self._make_request('GET', f'regions/{region_id}')
+
     def create_region(self, name, description):
         return self._make_request('POST', 'regions', {'name': name, 'description': description})
-    
-    def get_region_access(self, region_id):
-        return self._make_request('GET', f'region_access/by_region/{region_id}')
 
+    def update_region(self, region_id, data):
+        return self._make_request('PUT', f'regions/{region_id}', data)
+
+    def delete_region(self, region_id):
+        return self._make_request('DELETE', f'regions/{region_id}')
+
+    # Region Access CRUD
+    def get_region_accesses(self):
+        return self._make_request('GET', 'region_access')
+
+    def get_region_access(self, region_access_id):
+        return self._make_request('GET', f'region_access/{region_access_id}')
+
+    def create_region_access(self, data):
+        return self._make_request('POST', 'region_access', data)
+
+    def update_region_access(self, region_access_id, data):
+        return self._make_request('PUT', f'region_access/{region_access_id}', data)
+
+    def delete_region_access(self, region_access_id):
+        return self._make_request('DELETE', f'region_access/{region_access_id}')
+
+    # Workload Host CRUD
     def get_workload_hosts(self):
         return self._make_request('GET', 'workload_hosts')
 
-    def get_workload_host(self,workload_host_id):
+    def get_workload_host(self, workload_host_id):
         return self._make_request('GET', f'workload_hosts/{workload_host_id}')
-    
+
+    def create_workload_host(self, data):
+        return self._make_request('POST', 'workload_hosts', data)
+
+    def update_workload_host(self, workload_host_id, data):
+        return self._make_request('PUT', f'workload_hosts/{workload_host_id}', data)
+
+    def delete_workload_host(self, workload_host_id):
+        return self._make_request('DELETE', f'workload_hosts/{workload_host_id}')
+
+    # Image CRUD
     def get_images(self):
         return self._make_request('GET', 'images')
-    
-    def get_image(self,image_id):
+
+    def get_image(self, image_id):
         return self._make_request('GET', f'images/{image_id}')
-    
-    def edit_image(self,image_id,data):
-        return self._make_request('PUT', f'images/{image_id}',data)
 
     def create_image(self, data):
-        # Make the API request
-        return self._make_request('POST', 'images', data)   
-
+        print(":creating image")
+        return self._make_request('POST', 'images', data)
 
+    def update_image(self, image_id, data):
+        return self._make_request('PUT', f'images/{image_id}', data)
 
+    def delete_image(self, image_id):
+        return self._make_request('DELETE', f'images/{image_id}')
 
+    # Container CRUD
     def get_containers(self):
         return self._make_request('GET', 'workloads/containers')
-    
-    def get_container(self,container_id):
+
+    def get_container(self, container_id):
         return self._make_request('GET', f'workloads/containers/{container_id}')
-    
-    def edit_container(self,container_id,data):
-        return self._make_request('PUT', f'workloads/containers/{container_id}',data)
 
     def create_container(self, data):
-        # Make the API request
-        return self._make_request('POST', 'workloads/containers', data)   
+        return self._make_request('POST', 'workloads/containers', data)
+
+    def update_container(self, container_id, data):
+        return self._make_request('PUT', f'workloads/containers/{container_id}', data)
 
     def delete_container(self, container_id):
-        # Make the API request
-        return self._make_request('DELETE', f'workloads/containers/{container_id}')   
+        return self._make_request('DELETE', f'workloads/containers/{container_id}')
+
+    # Virtual Machine CRUD
+    def get_virtual_machines(self):
+        return self._make_request('GET', 'workloads/virtual_machines')
+
+    def get_virtual_machine(self, virtual_machine_id):
+        return self._make_request('GET', f'workloads/virtual_machines/{virtual_machine_id}')
+
+    def create_virtual_machine(self, data):
+        return self._make_request('POST', 'workloads/virtual_machines', data)
+
+    def update_virtual_machine(self, virtual_machine_id, data):
+        return self._make_request('PUT', f'workloads/virtual_machines/{virtual_machine_id}', data)
+
+    def delete_virtual_machine(self, virtual_machine_id):
+        return self._make_request('DELETE', f'workloads/virtual_machines/{virtual_machine_id}')
+
+    # Workload CRUD
+    def get_workloads(self):
+        return self._make_request('GET', 'workloads')
+
+    def get_workload(self, workload_id):
+        return self._make_request('GET', f'workloads/{workload_id}')
+
+    def create_workload(self, data):
+        return self._make_request('POST', 'workloads', data)
+
+    def update_workload(self, workload_id, data):
+        return self._make_request('PUT', f'workloads/{workload_id}', data)
+
+    def delete_workload(self, workload_id):
+        return self._make_request('DELETE', f'workloads/{workload_id}')
     
+
+
+    # Workload Host Resources
+    def get_pooled_resources_for_host(self, host_id):
+        """
+        Fetch pooled resources for a specific workload host.
+        """
+        return self._make_request('GET', f'workload-hosts/{host_id}/pooled-resources')
+
+    def get_fixed_resources_for_host(self, host_id):
+        """
+        Fetch fixed resources for a specific workload host.
+        """
+        return self._make_request('GET', f'workload-hosts/{host_id}/fixed-resources')
+
+    def get_ovs_bridges_for_host(self, host_id):
+        """
+        Fetch OVS bridges for a specific workload host.
+        """
+        return self._make_request('GET', f'workload-hosts/{host_id}/ovs-bridges')    
     
- 
+
+
+    # Workloads for Host
+    def get_workloads_for_host(self, host_id):
+        """
+        Fetch all workloads (containers and virtual machines) for a specific host.
+        """
+        # Fetch all containers and virtual machines
+        containers = self.get_containers() or []
+        virtual_machines = self.get_virtual_machines() or []
+
+        # Filter workloads by host_id
+        workloads = []
+        for container in containers:
+            if container.get('workload_host_id') == host_id:
+                container['workload_type'] = 'container'  # Add workload type
+                workloads.append(container)
+
+        for vm in virtual_machines:
+            if vm.get('workload_host_id') == host_id:
+                vm['workload_type'] = 'virtual_machine'  # Add workload type
+                workloads.append(vm)
+
+        return workloads    

app/controller/__init__.py

@@ -19,4 +19,6 @@ from .api.universe_routes import *
 from .api.workload_host_routes import *
 from .api.region_access_routes import *
 from .api.workload_container_routes import *
-from .api.image_routes import *
+from .api.image_routes import *
+from .api.workload_vm_routes import *
+from .api.virtual_data_center_routes import *

app/controller/api/api_routes.py

@@ -1,6 +1,6 @@
 from flask import request, jsonify
 from app.models.models import (
-    Universe, Project, VirtualDataCenter, Workload, Network, Region, RegionAccess, Volume, Image,
+    Universe, Project, VirtualDataCenter, Workload, Region, RegionAccess, Volume, Image,
     WorkloadHost, WorkloadHostFixedResource, WorkloadHostPooledResource, Label, User
 )
 from app import app, db
@@ -57,8 +57,7 @@ def create_routes(model, endpoint):
 # Register API routes for each model
 # create_routes(Universe, 'universes')
 # create_routes(Project, 'projects')
-create_routes(VirtualDataCenter, 'vdcs')
-# create_routes(Network, 'networks')
+# create_routes(VirtualDataCenter, 'vdcs')
 # create_routes(Region, 'regions')
 create_routes(Volume, 'volumes')
 # create_routes(Image, 'images')

app/controller/api/image_routes.py

@@ -41,6 +41,7 @@ def validate_image_data(data, is_update=False):
         'location_type': str,
         'os_family': str,
         'os_version': str,
+        'checksum': str,
     }
     for field, field_type in optional_fields.items():
         if field in data and not isinstance(data[field], field_type):
@@ -65,7 +66,8 @@ def add_image():
         size=validated_data['size'],
         location_type=validated_data.get('location_type'),
         os_family=validated_data.get('os_family'),
-        os_version=validated_data.get('os_version')
+        os_version=validated_data.get('os_version'),
+        checksum=validated_data.get('checksum')
     )
     db.session.add(instance)
     db.session.commit()

app/controller/api/network_routes.py

@@ -1,6 +1,6 @@
 from flask import request, jsonify, abort
 from app import app, db, logger
-from app.models.models import Network, NetworkPort
+from app.models.network import Network, NetworkPort
 from datetime import datetime
 from app.controller import api_bp
 import uuid
@@ -102,7 +102,7 @@ def edit_network(network_id):
 
 @api_bp.route('/networks/<network_id>', methods=['GET'])
 def get_network(network_id):
-    network = Network.query.get_or_404(network_id)
+    network = Network.query.get_or_404(uuid.UUID(network_id))
     return jsonify(network.to_json())
 
 @api_bp.route('/networks/<network_id>', methods=['DELETE'])

app/controller/api/project_routes.py

@@ -43,7 +43,7 @@ def edit_project(project_id):
 
 @api_bp.route('/projects/<project_id>', methods=['GET'])
 def get_project(project_id):
-    project = Project.query.get_or_404(project_id)
+    project = Project.query.get_or_404(uuid.UUID(project_id))
     return jsonify(project.to_json())
 
 

app/controller/api/region_routes.py

@@ -31,9 +31,12 @@ def edit_region(region_id):
 #  Can they see ALL the details or just the public data?
 @api_bp.route('/regions/<region_id>')
 def get_region(region_id):
-    region = Region.query.get_or_404(region_id)
-    return jsonify(region.to_json())
-
+    region = Region.query.get_or_404(uuid.UUID(region_id))
+    return jsonify({
+        'name': region.name,
+        'id': region.id,
+        'status': region.status,
+        })
 
 
 @app.route('/regions/<region_id>', methods=['DELETE'])

app/controller/api/universe_routes.py

@@ -37,7 +37,7 @@ def edit_universe(universe_id):
 
 @api_bp.route('/universes/<universe_id>', methods=['GET'])
 def get_universe(universe_id):
-    universe = Universe.query.get_or_404(universe_id)
+    universe = Universe.query.get_or_404(uuid.UUID(universe_id))
     return jsonify(universe.to_json())
 
 @api_bp.route('/universes/<universe_id>', methods=['DELETE'])

app/controller/api/virtual_data_center_routes.py

@@ -0,0 +1,53 @@
+from flask import request, jsonify
+from app import app, db, logger
+from app.models.models import VirtualDataCenter
+from app.controller import api_bp
+import uuid
+
+@api_bp.route('/virtual_data_centers', methods=['POST'])
+def add_vdc():
+    data = request.json
+    instance = VirtualDataCenter(
+        name=data['name'],
+        description=data.get('description'),
+        status=data.get('status'),
+        created_by=data.get('created_by')
+    )
+    db.session.add(instance)
+    db.session.commit()
+    logger.debug("vdc added to DB")
+    return jsonify(instance.to_json()), 201
+
+@api_bp.route('/virtual_data_centers/<virtual_data_center_id>', methods=['PUT'])
+def edit_vdc(virtual_data_center_id):
+    vdc = VirtualDataCenter.query.get_or_404(virtual_data_center_id)
+    data = request.json
+    
+    if 'name' in data:
+        vdc.name = data['name']
+    if 'description' in data:
+        vdc.description = data['description']
+    if 'status' in data:
+        vdc.status = data['status']
+    if 'visible' in data:
+        vdc.visible = data['visible']
+    
+    db.session.commit()
+    return jsonify(success=True)
+
+@api_bp.route('/virtual_data_centers/<virtual_data_center_id>', methods=['GET'])
+def get_virtual_data_center(virtual_data_center_id):
+    vdc = VirtualDataCenter.query.get_or_404(uuid.UUID(virtual_data_center_id))
+    return jsonify(vdc.to_json())
+
+@api_bp.route('/virtual_data_centers/<virtual_data_center_id>', methods=['DELETE'])
+def delete_vdc(virtual_data_center_id):
+    vdc = VirtualDataCenter.query.get_or_404(virtual_data_center_id)
+    vdc.soft_delete()
+    db.session.commit()
+    return jsonify({'message': 'vdc deleted successfully'}), 200
+
+@api_bp.route('/virtual_data_centers', methods=['GET'])
+def get_virtual_data_centers():
+    virtual_data_centers = VirtualDataCenter.query.filter_by(deleted=False).all()
+    return jsonify([vdc.to_json() for vdc in virtual_data_centers])

app/controller/api/workload_container_routes.py

@@ -90,7 +90,6 @@ def validate_payload(payload):
 def add_container_workload():
     request_data = request.json
     logger.debug(request_data)
-    all_containers=[]
 
     # Validate incoming data
     validated_data = validate_payload(request_data)
@@ -115,42 +114,43 @@ def add_container_workload():
         # TODO - This is where we would slot in the 'placement' routine
         random_host = random.choice(all_workload_hosts)
         logger.info(f"Randomly selected host: {random_host}")
-
+        NSController=None
 
         # Check if we need to deploy an NS Controller to this host for this VPC ID or not
         # TODO - We will need to update the worker code too.
         #  - If the workder doesnt have an NS controller present for the given VPCID(Currently called a tennacny) then fail the request
         # Lets try and add the NSController deployment request into the same payload
-        ns_controller_query=Workload.query.filter_by(
+        existing_NS_Controller=Workload.query.filter_by(
             workload_host_id=random_host.id,
             workload_type="NSController",  # Always set workload_type to "Container"
             vdc_id=request_vdc.id,
             deleted=0
-        ).all()
+        ).first()
 
-        if len(ns_controller_query)==0:
+        if not existing_NS_Controller:
+            # No Namespace controller exists on this host, lets create one
             logger.warning(f"No NSController found in on host {random_host.name} for VDC {request_vdc.name}, adding it to the payload")
             # TODO - we need to assign a network port to the NS Controller including assigning an IP - How do we specify this in the payload?
+            new_NSController_name=f"NSCONTROLLER_{request_vdc.id}_{random_host.id}"
             new_NSController = Workload(
-                        name=_container['container_name'],
+                        name=new_NSController_name,
                         workload_type="NSController",  # Always set workload_type to "Container"
                         vdc_id=request_vdc.id,
                         status="pending-allocation",
                         launch_params=json.dumps({
                             "docker_image": "busybox",
-                            "container_name": f"NSCONTROLLER-{request_vdc.id}"
+                            "container_name": new_NSController_name,
+                            "command": "sleep infinite"
                         }),
                         workload_host_id=random_host.id
                     )
             db.session.add(new_NSController)
             db.session.commit()
-    
-
-            NSController_launchparams=json.loads(new_NSController.launch_params)
-            NSController_launchparams['container_id']=new_NSController.id
-            all_containers.append(NSController_launchparams)
-
+            NSController=new_NSController
+        else:
+            NSController=existing_NS_Controller
 
+            
         # Create the Workload instance in the database
         new_container = Workload(
             name=_container['container_name'],
@@ -164,17 +164,19 @@ def add_container_workload():
         db.session.commit()
         logger.debug(f"Container {new_container.id} workload added to DB")
 
-        #Inject the container ID so that it can be processed by the workloadHost
+        # Inject the container ID so that it can be processed by the workloadHost
         _container['container_id']=new_container.id
-        logger.error(_container)
+        _container['NSController_launchparams']=json.loads(NSController.launch_params)
+        _container['NSController_launchparams']['container_id']=NSController.id
+        
+        logger.debug(f"Container spec: {_container}")
         #Send the request off to the websocket server to have this container launched
-        all_containers.append(_container)
         payload = {
             "worker_id": random_host.id,
             "task_type": "container-create",
             "job_details": {
                 "tenancyID": 3,
-                "containers": json.dumps(all_containers)
+                "container": json.dumps(_container)
             }
         }
         logger.debug(f"Sending payload to websocket server {payload}")
@@ -185,11 +187,16 @@ def add_container_workload():
 
         if websocket_server_response.status_code == 201:
             logger.info(f"Task created successfully for container {new_container.id}")
+            NSController.status="pending-allocated"
+            db.session.add(NSController)
+            
             new_container.status="pending-allocated"
             db.session.add(new_container)
             db.session.commit()
         else:
             logger.error(f"Creation request failed for container {new_container.id}")
+            NSController.status="pending-allocated"
+            db.session.add(NSController)
             new_container.status="failed-allocation"
             db.session.add(new_container)
             db.session.commit()
@@ -274,11 +281,10 @@ def delete_container_workload(workload_id):
         "worker_id": _container.workload_host_id,
         "task_type": "container-delete",
         "job_details": {
-            "tenancyID": 3,
-            "containers": [
+            "container": [
                 {
                     "container_id": _container.id,
-                    "deleted": "True"
+                    "desired_state": "deleted"
                 }
             ]
         }

app/controller/api/workload_vm_routes.py

@@ -0,0 +1,555 @@
+import random
+from flask import json, request, jsonify, abort
+import requests
+from app import app, db, logger
+from app.models.models import Volume, Workload, WorkloadHost, VirtualDataCenter, Image
+from datetime import datetime
+from app.controller import api_bp
+import uuid
+
+import uuid
+from werkzeug.exceptions import abort
+from sqlalchemy import or_
+
+from app.models.network import Network
+
+websocket_server_url = "http://127.0.0.1:6000/api/create_task"
+
+def validate_port_entry(port_entry):
+    """
+    Validate a single port entry in the open_ports list.
+    Valid formats include:
+    - protocol/port (e.g., "tcp/80")
+    - protocol/port-range (e.g., "tcp/1024-2048")
+    - protocol/port:source_ip (e.g., "tcp/22:103.104.105.106")
+    - protocol name only (e.g., "icmp", "tcp", "udp")
+    
+    Args:
+        port_entry (str): The port entry to validate
+        
+    Returns:
+        bool: True if valid, False otherwise
+    """
+    import re
+    
+    # Protocol names only (e.g., "icmp", "tcp", "udp")
+    if port_entry.lower() in ["icmp", "tcp", "udp"]:
+        return True
+    
+    # For other formats, split by protocol and the rest
+    if "/" not in port_entry:
+        return False
+    
+    protocol, rest = port_entry.split("/", 1)
+    
+    # Validate protocol
+    if protocol.lower() not in ["tcp", "udp", "icmp"]:
+        return False
+    
+    # Check for source IP format (port:ip)
+    if ":" in rest:
+        port_part, ip_part = rest.split(":", 1)
+        
+        # Validate IP address format
+        ip_pattern = r"^(\d{1,3}\.){3}\d{1,3}$"
+        if not re.match(ip_pattern, ip_part):
+            return False
+        
+        # Also validate that IP octets are in valid range (0-255)
+        octets = ip_part.split(".")
+        if not all(0 <= int(octet) <= 255 for octet in octets):
+            return False
+    else:
+        port_part = rest
+    
+    # Validate port or port range
+    if "-" in port_part:
+        # Port range format
+        try:
+            start_port, end_port = port_part.split("-", 1)
+            start_port = int(start_port)
+            end_port = int(end_port)
+            
+            # Validate port range
+            if not (0 <= start_port <= 65535 and 0 <= end_port <= 65535 and start_port < end_port):
+                return False
+        except ValueError:
+            return False
+    else:
+        # Single port format
+        try:
+            port = int(port_part)
+            if not (0 <= port <= 65535):
+                return False
+        except ValueError:
+            return False
+    
+    return True
+
+def validate_payload(payload):
+    """
+    Validate the input payload according to the specified rules and return a sanitized version.
+    
+    Args:
+        payload (dict): The input JSON object to validate.
+    
+    Returns:
+        dict: A sanitized version of the input payload containing only the expected parameters.
+    
+    Raises:
+        ValueError: If the payload fails any validation rule.
+    """
+    import uuid  # Added import for UUID validation
+    
+    # Ensure the input is a valid JSON object (dict)
+    if not isinstance(payload, dict):
+        raise ValueError("Input must be a valid JSON object (dict).")
+    
+    # Check for the top-level 'virtual_data_center' key and validate it as a UUID
+    if 'virtual_data_center' not in payload:
+        raise ValueError("Top-level 'virtual_data_center' key is missing.")
+    try:
+        uuid.UUID(payload['virtual_data_center'], version=4)
+    except ValueError:
+        raise ValueError("'virtual_data_center' must be a valid UUID.")
+    
+    # Check for the 'virtual-machines' key and ensure it is a list with at least one VM
+    if 'virtual-machines' not in payload:
+        raise ValueError("'virtual-machines' key is missing.")
+    if not isinstance(payload['virtual-machines'], list) or len(payload['virtual-machines']) == 0:
+        raise ValueError("'virtual-machines' must be a list with at least one VM.")
+    
+    # Initialize the sanitized payload
+    sanitized_payload = {
+        'virtual_data_center': payload['virtual_data_center'],
+        'virtual-machines': []
+    }
+    
+    # Validate each VM in the 'virtual-machines' list and build the sanitized version
+    for vm in payload['virtual-machines']:
+        if not isinstance(vm, dict):
+            raise ValueError("Each virtual machine must be a dictionary.")
+        
+        # Check for required keys in the new format
+        if 'name' not in vm:
+            raise ValueError("Virtual machine is missing 'name'.")
+        if 'memory' not in vm:
+            raise ValueError("Virtual machine is missing 'memory'.")
+        if 'vcpu' not in vm:
+            raise ValueError("Virtual machine is missing 'vcpu'.")
+        
+        # Validate numeric fields
+        if not isinstance(vm['memory'], int) or vm['memory'] <= 0:
+            raise ValueError("'memory' must be a positive integer.")
+        if not isinstance(vm['vcpu'], int) or vm['vcpu'] <= 0:
+            raise ValueError("'vcpu' must be a positive integer.")
+        
+        # Initialize the sanitized VM
+        sanitized_vm = {
+            'name': vm['name'],
+            'memory': vm['memory'],
+            'vcpu': vm['vcpu']
+        }
+        
+        # Validate 'volumes' if present
+        if 'volumes' in vm:
+            if not isinstance(vm['volumes'], list):
+                raise ValueError("'volumes' must be a list.")
+            
+            sanitized_volumes = []
+            for volume in vm['volumes']:
+                if not isinstance(volume, dict):
+                    raise ValueError("Each volume must be a dictionary.")
+                
+                if 'name' not in volume or 'size_gb' not in volume:
+                    raise ValueError("Volume is missing 'name' or 'size_gb'.")
+
+                sanitized_volume = {
+                    'name': volume['name'],
+                    'size_gb': volume['size_gb'],
+                }
+
+
+                if not isinstance(volume['size_gb'], (int, float)) or volume['size_gb'] <= 0:
+                    raise ValueError("'size_gb' must be a positive number.")
+                # Check if 'image' is present and validate it as a UUID
+                if 'image_id' in volume:
+                    try:
+                        uuid.UUID(volume['image_id'])
+                    except ValueError:
+                        raise ValueError(f"'{volume['image_id']}' is not a valid UUID.")
+                    sanitized_volume['image_id'] = volume['image_id']
+               
+                # Add optional volume fields if present
+                if 'boot' in volume:
+                    if not isinstance(volume['boot'], bool):
+                        raise ValueError("'boot' must be a boolean.")
+                    sanitized_volume['boot'] = volume['boot']
+                
+                if 'type' in volume:
+                    sanitized_volume['type'] = volume['type']
+                
+                if 'source' in volume:
+                    sanitized_volume['source'] = volume['source']
+        
+
+                sanitized_volumes.append(sanitized_volume)
+            
+            sanitized_vm['volumes'] = sanitized_volumes
+        
+        # Validate 'networks' if present
+        if 'networks' in vm:
+            if not isinstance(vm['networks'], list):
+                raise ValueError("'networks' must be a list.")
+            
+            sanitized_networks = []
+            for network in vm['networks']:
+                if not isinstance(network, dict):
+                    raise ValueError("Each network must be a dictionary.")
+                
+                if 'id' not in network:
+                    raise ValueError("Network is missing 'id'.")
+                
+                sanitized_network = {
+                    'id': network['id']
+                }
+                
+                # Add optional network fields if present
+                if 'mac' in network:
+                    # Could add MAC address format validation here
+                    sanitized_network['mac'] = network['mac']
+                
+                if 'ip_address' in network:
+                    # Could add IP address format validation here
+                    sanitized_network['ip_address'] = network['ip_address']
+                
+                if 'open_ports' in network:
+                    if not isinstance(network['open_ports'], list):
+                        raise ValueError("'open_ports' must be a list.")
+                    
+                    # Validate each port entry
+                    for port_entry in network['open_ports']:
+                        if not isinstance(port_entry, str):
+                            raise ValueError("Each port entry must be a string.")
+                        
+                        if not validate_port_entry(port_entry):
+                            raise ValueError(f"Invalid port entry format: '{port_entry}'")
+                    
+                    sanitized_network['open_ports'] = network['open_ports']
+                
+                sanitized_networks.append(sanitized_network)
+            
+            sanitized_vm['networks'] = sanitized_networks
+        
+        # Validate 'tags' if present
+        if 'tags' in vm:
+            if not isinstance(vm['tags'], dict):
+                raise ValueError("'tags' must be a dictionary.")
+            sanitized_vm['tags'] = vm['tags']
+        
+        # Validate 'requirements' if present
+        if 'requirements' in vm:
+            if not isinstance(vm['requirements'], dict):
+                raise ValueError("'requirements' must be a dictionary.")
+            sanitized_vm['requirements'] = vm['requirements']
+        
+        # Add the sanitized VM to the sanitized payload
+        sanitized_payload['virtual-machines'].append(sanitized_vm)
+    
+    return sanitized_payload
+
+@api_bp.route('/workloads/virtual_machines', methods=['POST'])
+def add_VirtualMachine_workload():
+    request_data = request.json
+    logger.debug(f"Request data: {request_data}")
+
+    # Validate incoming data
+    validated_data = validate_payload(request_data)
+    
+    # TODO - Check if this user has permissions to VIEW and LAUNCH_VirtualMachine in this VDC
+    request_vdc = VirtualDataCenter.query.filter_by(id=uuid.UUID(validated_data['virtual_data_center'])).first()
+
+    if request_vdc is None:
+        # Raise an error or return a response indicating that no VDC was found
+        abort(404, description="Virtual Data Center not found")  # Flask's abort function
+
+    all_workload_hosts = WorkloadHost.query.filter_by(deleted=0, region_id=request_vdc.region.id).all()
+    logger.debug(f"Found {len(all_workload_hosts)} hosts eligible for placement")
+
+    if len(all_workload_hosts) == 0:
+        logger.warning(f"No hosts found in region {request_vdc.region.name} to place this request")
+        return f"No hosts found in region {request_vdc.region.name} to place this request", 400
+    
+    client_response = []
+    for VirtualMachine in validated_data['virtual-machines']:
+        # TODO - This is where we would slot in the 'placement' routine
+        random_host = random.choice(all_workload_hosts)
+        logger.info(f"Randomly selected host: {random_host}")
+
+        # Create the Workload instance in the database
+        new_VirtualMachine = Workload(
+            name=VirtualMachine['name'],
+            workload_type="VirtualMachine",  # Set workload_type to VirtualMachine
+            vdc_id=request_vdc.id,
+            status="pending-allocation",
+            launch_params=json.dumps(VirtualMachine ),
+            workload_host_id=random_host.id
+        )
+        db.session.add(new_VirtualMachine)
+        db.session.commit()
+        logger.debug(f"VirtualMachine {new_VirtualMachine.id} workload added to DB")
+        
+        # Create volumes in the database
+        volumes = []
+        for volume in VirtualMachine.get('volumes', []):
+            logger.debug(f"Volume details: {volume}")
+            _volume_source=""
+            _volume_image_id=None
+
+            if "image_id" in volume:
+                logger.debug(f"Volume had image id specified {volume['image_id']}, confirming if it's a legitimate image ID or not")
+                
+                # Validate the image ID
+                try:
+                    # Convert the image ID to a UUID object
+                    image_id = uuid.UUID(volume['image_id'])
+                    
+                    # Query the image from the database
+                    _image = Image.query.filter_by(id=image_id).first()
+                    
+                    if _image is None:
+                        # If the image does not exist, return an error
+                        return {"error": f"Image with ID {volume['image_id']} does not exist"}, 404
+                    logger.debug("Image seems legit")
+                    # If the image exists, load the image ID into a variable
+                    _volume_image_id = _image.id
+                    _volume_source = _image.location
+                    logger.debug(f"Image ID {image_id} is valid and loaded.")
+                    
+                except ValueError:
+                    # Handle the case where the image ID is not a valid UUID
+                    return {"error": f"Invalid image ID format: {volume['image_id']}"}, 400
+            else:
+                logger.debug(f"No image specified for this volume")
+
+            logger.debug(f"Vol type is {volume['type']}")
+            _volume_path=volume['type']
+
+            new_volume = Volume(
+                name=volume['name'],
+                image_id=_volume_image_id,
+                type=_volume_path,
+                path=_volume_path,
+                size_gb=volume['size_gb'],
+                vdc_id=request_vdc.id,
+                source=_volume_source
+            )
+            logger.debug(f"Volume spec: {new_volume.to_json()}")
+            db.session.add(new_volume)
+            db.session.commit()
+            volumes.append(new_volume)
+
+        # Create network ports if networks are specified
+        networks = []
+        for network in VirtualMachine.get('networks', []):
+            network_obj = Network.query.filter_by(id=uuid.UUID(network['id'])).first()
+            if network_obj:
+                port = network_obj.create_port(db.session, workload_id=new_VirtualMachine.id)
+                networks.append(port)
+
+        # Send the request off to the websocket server to have this VirtualMachine launched
+        payload = {
+            "worker_id": random_host.id,
+            "task_type": "virtual-machine-create",
+            "job_details": {
+                "virtual_machine_name": VirtualMachine['name'],
+                "virtual_machine_id": new_VirtualMachine.id,
+                "desired_state": "running",
+                "virtual_machine_config": {
+                    "memory": VirtualMachine['memory'],
+                    "vcpu": VirtualMachine['vcpu'],
+                    "volumes": [
+                        {
+                            "id": vol.id,
+                            "name": vol.name,
+                            "size_gb": vol.size_gb,
+                            "type": vol.type,
+                            "path": vol.path,
+                            **({"source": {  # Include image object only if image_id exists
+                                "type": "image",
+                                "image_id": vol.image.id,
+                                "path": vol.image.location,
+                                "format": vol.image.format,
+                                "checksum": vol.image.checksum
+                            }} if vol.image_id else {
+                                "source": "None"
+                            })  # Conditionally add image object
+                        }
+                        for vol in volumes
+                    ],
+                    "networks": [
+                        {
+                            "id": net.id,
+                            "mac_address": net.mac_address
+                        }
+                        for net in networks
+                    ]
+                }
+            }
+        }
+        logger.debug(f"Sending payload to websocket server {payload}")
+        headers = {"Content-Type": "application/json"}
+        
+        websocket_server_response = requests.post(websocket_server_url, data=json.dumps(payload), headers=headers)
+        websocket_server_response_data = websocket_server_response.json()
+
+        if websocket_server_response.status_code == 201:
+            logger.info(f"Task created successfully for VirtualMachine {new_VirtualMachine.id}")
+            new_VirtualMachine.status = "pending-allocated"
+            db.session.add(new_VirtualMachine)
+            db.session.commit()
+        else:
+            logger.error(f"Creation request failed for VirtualMachine {new_VirtualMachine.id}")
+            new_VirtualMachine.status = "failed-allocation"
+            db.session.add(new_VirtualMachine)
+            db.session.commit()
+
+        # Prepare the response to the API request
+        client_response.append({
+            "virtual_machine_id": new_VirtualMachine.id,
+            "virtual_machine_name": new_VirtualMachine.name,
+            "VirtualMachine_status": new_VirtualMachine.status,
+        })
+
+    return client_response, 200
+
+
+@api_bp.route('/workloads/virtual_machines/status_update/<virtual_machine_id>', methods=['PUT'])
+def edit_VirtualMachine_workload(virtual_machine_id):
+    data = request.json
+
+    # Validate new_status
+    if data.get("new_status") not in ["running", "deleted", "stopped"]:
+        error_message = "Invalid status. Must be 'running' or 'deleted'."
+        logger.error(error_message)
+        return jsonify({"success": False, "message": error_message}), 400
+
+    # Validate system_vm_id and worker_id as UUIDs
+    try:
+        worker_id = uuid.UUID(data.get("worker_id"))
+    except (ValueError, TypeError) as e:
+        error_message = "Invalid UUID format for worker_id."
+        logger.error(f"{error_message} Error: {str(e)}")
+        return jsonify({"success": False, "message": error_message}), 400
+
+    # Validate virtual_machine_id as UUID
+    try:
+        virtual_machine_id = uuid.UUID(virtual_machine_id)
+    except (ValueError, TypeError) as e:
+        error_message = "Invalid UUID format for virtual_machine_id."
+        logger.error(f"{error_message} Error: {str(e)}")
+        return jsonify({"success": False, "message": error_message}), 400
+
+    # Validate timestamp as ISO format
+    try:
+        datetime.fromisoformat(data.get("timestamp"))
+    except (ValueError, TypeError) as e:
+        error_message = "Invalid ISO timestamp format."
+        logger.error(f"{error_message} Error: {str(e)}")
+        return jsonify({"success": False, "message": error_message}), 400
+
+    # Fetch the virtual machine from the database
+    try:
+        virtual_machine = Workload.query.filter_by(id=virtual_machine_id, workload_type="VirtualMachine").first_or_404()
+    except Exception as e:
+        error_message = f"Failed to fetch virtual machine with ID {virtual_machine_id}."
+        logger.error(f"{error_message} Error: {str(e)}")
+        return jsonify({"success": False, "message": error_message}), 404
+
+    # Check if the VM is actually on the reported worker
+    if virtual_machine.workload_host_id != worker_id:
+        error_message = f"Workload host in DB for VM ID {virtual_machine_id} is {virtual_machine.workload_host_id}, but was reported from {worker_id}. Ignoring."
+        logger.error(error_message)
+        return jsonify({"success": False, "message": error_message}), 400
+
+    # Update the status of the virtual machine
+    try:
+        new_status = data.get('new_status')
+        logger.info(f"Updating VirtualMachine ID: {virtual_machine_id} to new status {new_status}")
+        virtual_machine.status = new_status
+
+        # If new status = 'started' 
+        db.session.commit()
+        return jsonify({"success": True, "message": "Status updated successfully."}), 200
+    except Exception as e:
+        error_message = f"Failed to update status for VirtualMachine ID: {virtual_machine_id}."
+        logger.error(f"{error_message} Error: {str(e)}")
+        db.session.rollback()
+        return jsonify({"success": False, "message": error_message}), 500
+    
+
+@api_bp.route('/workloads/virtual_machines/<workload_id>', methods=['GET'])
+def get_VirtualMachine_workload(workload_id):
+    try:
+        workload_uuid = uuid.UUID(workload_id)
+    except ValueError:
+        return jsonify({"error": "Invalid workload ID"}), 404
+    
+    workload = Workload.query.filter(
+        Workload.id == workload_uuid,
+        Workload.workload_type == "VirtualMachine",
+        Workload.deleted == False
+    ).first_or_404()
+    logger.info(f"{workload.workload_type} {workload.deleted}")
+    return jsonify(workload.to_json())
+
+@api_bp.route('/workloads/virtual_machines/<workload_id>', methods=['DELETE'])
+def delete_VirtualMachine_workload(workload_id):
+    try:
+        workload_uuid = uuid.UUID(workload_id)
+    except ValueError:
+        return jsonify({"error": "Invalid workload ID"}), 404
+    
+    _VirtualMachine = Workload.query.filter(
+        Workload.id == workload_uuid,
+        Workload.workload_type == "VirtualMachine",
+        Workload.deleted == False
+    ).first_or_404()
+    
+    payload = {
+        "worker_id": _VirtualMachine.workload_host_id,
+        "task_type": "virtual-machine-delete",
+        "job_details": {
+            "virtual_machine_id": _VirtualMachine.id,
+            "desired_state": "deleted"
+        }
+    }
+    logger.debug(f"Sending payload to websocket server {payload}")
+    headers = {"Content-Type": "application/json"}
+    
+    websocket_server_response = requests.post(websocket_server_url, data=json.dumps(payload), headers=headers)
+    websocket_server_response_data = websocket_server_response.json()
+    
+    if websocket_server_response.status_code == 201:
+        logger.info(f"Task created successfully for VirtualMachine {_VirtualMachine.id}")
+        _VirtualMachine.status = "pending-deleted"
+        db.session.add(_VirtualMachine)
+        db.session.commit()
+    else:
+        logger.error(f"Creation request failed for VirtualMachine {_VirtualMachine.id}")
+        _VirtualMachine.status = "failed-deleted"
+        db.session.add(_VirtualMachine)
+        db.session.commit()
+    
+    _VirtualMachine.soft_delete()
+    logger.info(f"Deleted VirtualMachine {_VirtualMachine.id}")
+    return jsonify({'message': 'VirtualMachine workload deleted successfully'}), 200
+
+@api_bp.route('/workloads/virtual_machines', methods=['GET'])
+def get_VirtualMachine_workloads():
+    workloads = Workload.query.filter(
+        Workload.workload_type == "VirtualMachine",
+        Workload.deleted == False
+    ).all()
+    logger.debug(f"Found {len(workloads)} VirtualMachines")
+    return jsonify([workload.to_json() for workload in workloads])

app/controller/ui_routes.py

@@ -1,7 +1,8 @@
 from flask import request, jsonify, render_template, g, url_for
-from app.models.models import Universe, Project, VirtualDataCenter, Workload, Network, Region
+import requests
+from app.models.models import Universe, Project, VirtualDataCenter, Workload, Region
 from app.controller import ui_bp
-
+from app.models.network import *
 # UI Routes with  prefix
 
 @ui_bp.route('/')
@@ -37,3 +38,27 @@ def networks():
 def regions():
     regions = Region.query.all()
     return render_template('regions.html', regions=regions)
+
+
+@ui_bp.route("/vnc/<vm_id>")
+def console(vm_id):
+    """Render the noVNC client page for a specific VM"""
+    # Fetch the WebSocket URL from the API
+    API_SERVER_URL = "http://localhost:5000"  # API to get VNC session token
+
+    # response = requests.post(f"{API_SERVER_URL}/get-console-url", json={"username": "test_user", "worker_id": vm_id})
+
+    # if response.status_code != 200:
+    #     return "Error: Unable to get VNC session", 500
+
+    # console_data = response.json()
+    # console_url = console_data.get("console_url")
+    # Get VNC connection parameters from request args or use defaults
+    websocket_host = '127.0.0.1'
+    websocket_port = 6080  # Standard noVNC port
+    path = ''
+    
+    return render_template('console.html', 
+                          websocket_host=websocket_host,
+                          websocket_port=websocket_port,
+                          path=path)

app/models/models.py

@@ -133,34 +133,10 @@ class Workload(BaseModel):
     status = Column(String, nullable=True)
     container_id = Column(String, nullable=True)
     launch_params = Column(String, nullable=True)
+    time_to_provision = Column(Float, nullable=True)
 
-class Network(BaseModel):
-    __tablename__ = "networks"
-    vdc_id = Column(UUID(as_uuid=True), ForeignKey("vdcs.id"), nullable=False)
-    ipv4_cidr = Column(String, nullable=True)
-    ipv4_gateway = Column(String, nullable=True)
-    ipv4_dns_servers = Column(String, nullable=True)
-    ipv6_cidr = Column(String, nullable=True)
-    ipv6_gateway = Column(String, nullable=True)
-    vni = Column(Integer, nullable=False)
-    encapsulation = Column(String, default="VXLAN", nullable=False)
-    ovs_bridge = Column(String, nullable=True)
-
-
-class NetworkPort(BaseModel):
-    __tablename__ = "network_ports"
-
-    # Foreign Keys
-    network_id = Column(UUID(as_uuid=True), ForeignKey("networks.id"), nullable=False)
-    workload_id = Column(UUID(as_uuid=True), ForeignKey("workloads.id"), nullable=True)
-
-    # Network Port Properties
-    name = Column(String, nullable=True)
-    ip_address = Column(String, nullable=True)  # IPv4 or IPv6 address
-    mac_address = Column(String, nullable=True)  # MAC address
-    port_type = Column(String, nullable=True)  # e.g., "public", "private", "management"
-    dns_servers = Column(String, nullable=True)  # DNS servers (comma-separated)
-    subnet_mask = Column(String, nullable=True)  # Subnet mask for the IP address
+    # Relationship to VolumeWorkloadMapping
+    volume_mappings = relationship("VolumeWorkloadMapping", back_populates="workload")
 
 class Image(BaseModel):
     __tablename__ = "images"
@@ -169,13 +145,39 @@ class Image(BaseModel):
     size = Column(Float, nullable=False)
     os_family = Column(String, nullable=True)
     os_version = Column(String, nullable=True)
+    checksum = Column(String, nullable=True)
+    format = Column(String, nullable=True)
+    # Relationship to Volume
+    volumes = relationship("Volume", back_populates="image")
 
 class Volume(BaseModel):
     __tablename__ = "volumes"
-    volume_type = Column(Enum("Ceph", "Local", name="volume_type"), nullable=False)
-    path = Column(String, nullable=False)
+    path = Column(String, nullable=True)
+    type = Column(String, nullable=True)
+    source = Column(String, nullable=True)
     size_gb = Column(Float, nullable=False)
+    boot = Column(Boolean, nullable=False, default=False)
     vdc_id = Column(UUID(as_uuid=True), ForeignKey("vdcs.id"), nullable=False)
+    # Optional foreign key to link a volume to an image
+    image_id = Column(UUID(as_uuid=True), ForeignKey("images.id"), nullable=True)
+    
+    # Relationship to Image
+    image = relationship("Image", back_populates="volumes")
+    
+    # Relationship to VolumeWorkloadMapping
+    workload_mappings = relationship("VolumeWorkloadMapping", back_populates="volume")
+
+class VolumeWorkloadMapping(BaseModel):
+    __tablename__ = "volume_workload_mapping"
+    
+    # Composite primary key (volume_id and workload_id)
+    volume_id = Column(UUID(as_uuid=True), ForeignKey("volumes.id"), primary_key=True)
+    workload_id = Column(UUID(as_uuid=True), ForeignKey("workloads.id"), primary_key=True)
+    
+    # Relationships
+    volume = relationship("Volume", back_populates="workload_mappings")
+    workload = relationship("Workload", back_populates="volume_mappings")
+
 
 class User(db.Model):
     __tablename__ = "users"

app/models/network.py

@@ -0,0 +1,82 @@
+import uuid
+import ipaddress
+from sqlalchemy.orm import Session
+
+from datetime import datetime
+from sqlalchemy import (
+    Column, String, Boolean, DateTime, ForeignKey, Integer, Float, Text, Enum
+)
+from sqlalchemy.dialects.postgresql import UUID
+
+from .models import BaseModel
+
+
+class NetworkPort(BaseModel):
+    __tablename__ = "network_ports"
+
+    # Foreign Keys
+    network_id = Column(UUID(as_uuid=True), ForeignKey("networks.id"), nullable=False)
+    workload_id = Column(UUID(as_uuid=True), ForeignKey("workloads.id"), nullable=True)
+
+    # Network Port Properties
+    name = Column(String, nullable=True)
+    ip_address = Column(String, nullable=True)  # IPv4 or IPv6 address
+    mac_address = Column(String, nullable=True)  # MAC address
+    port_type = Column(String, nullable=True)  # e.g., "public", "private", "management"
+    dns_servers = Column(String, nullable=True)  # DNS servers (comma-separated)
+    subnet_mask = Column(String, nullable=True)  # Subnet mask for the IP address
+
+
+
+class Network(BaseModel):
+    __tablename__ = "networks"
+    vdc_id = Column(UUID(as_uuid=True), ForeignKey("vdcs.id"), nullable=False)
+    ipv4_cidr = Column(String, nullable=True)
+    ipv4_gateway = Column(String, nullable=True)
+    ipv4_dns_servers = Column(String, nullable=True)
+    ipv6_cidr = Column(String, nullable=True)
+    ipv6_gateway = Column(String, nullable=True)
+    vni = Column(Integer, nullable=False)
+    encapsulation = Column(String, default="VXLAN", nullable=False)
+    ovs_bridge = Column(String, nullable=True)
+    
+    def create_port(self, db: Session, workload_id: UUID = None, name: str = None, port_type: str = None):
+        # Define the prefix
+        prefix = "fa:16:4a"
+
+        # Generate the MAC address with the specified prefix
+        mac_address = prefix + ':' + ':'.join(f'{byte:02x}' for byte in uuid.uuid4().bytes[-3:] )
+
+        # Generate a unique IP address within the network's CIDR range
+        ip_network = ipaddress.ip_network(self.ipv4_cidr)
+        used_ips = {port.ip_address for port in db.query(NetworkPort).filter(NetworkPort.network_id == self.id).all()}
+        
+        for ip in ip_network.hosts():
+            if str(ip) not in used_ips:
+                ip_address = str(ip)
+                break
+        else:
+            raise ValueError("No available IP addresses in the network")
+
+        # Check for MAC address conflicts
+        if db.query(NetworkPort).filter(NetworkPort.mac_address == mac_address).first():
+            raise ValueError("MAC address conflict detected")
+
+        # Create the new port
+        new_port = NetworkPort(
+            network_id=self.id,
+            workload_id=workload_id,
+            name=name,
+            ip_address=ip_address,
+            mac_address=mac_address,
+            port_type=port_type,
+            dns_servers=self.ipv4_dns_servers,
+            subnet_mask=str(ip_network.netmask),
+        )
+        
+        db.add(new_port)
+        db.commit()
+        db.refresh(new_port)
+
+        return new_port
+

app/static/style.css

@@ -4,3 +4,60 @@ body {
 h2 {
     margin-bottom: 20px;
 }
+body {
+    margin: 0;
+    padding: 0;
+    height: 100vh;
+    display: flex;
+    flex-direction: column;
+    font-family: Arial, sans-serif;
+}
+
+#novnc-container {
+    flex-grow: 1;
+    background-color: #f0f0f0;
+    position: relative;
+}
+
+#vnc-canvas {
+    width: 100%;
+    height: 100%;
+}
+
+#connection-status {
+    position: absolute;
+    top: 10px;
+    left: 10px;
+    padding: 5px 10px;
+    background-color: rgba(0, 0, 0, 0.7);
+    color: white;
+    border-radius: 5px;
+    z-index: 100;
+}
+
+.controls {
+    padding: 10px;
+    background-color: #eaeaea;
+    border-bottom: 1px solid #ccc;
+    display: flex;
+    justify-content: space-between;
+}
+
+button {
+    padding: 6px 12px;
+    margin: 0 4px;
+    background-color: #4CAF50;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+}
+
+button:hover {
+    background-color: #45a049;
+}
+
+button:disabled {
+    background-color: #cccccc;
+    cursor: not-allowed;
+}

app/templates/console.html

@@ -0,0 +1,416 @@
+<!DOCTYPE html>
+<html lang="en" class="noVNC_loading">
+<head>
+    <!--
+    noVNC example: simple example using default UI
+    Copyright (C) 2019 The noVNC authors
+    noVNC is licensed under the MPL 2.0 (see LICENSE.txt)
+    This file is licensed under the 2-Clause BSD license (see LICENSE.txt).
+
+    Connect parameters are provided in query string:
+        http://example.com/?host=HOST&port=PORT&encrypt=1
+    or the fragment:
+        http://example.com/#host=HOST&port=PORT&encrypt=1
+    -->
+    <title>noVNC</title>
+
+    <link rel="icon" type="image/x-icon" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc.ico') }}">
+    <meta name="theme-color" content="#313131">
+
+    <!-- Apple iOS Safari settings -->
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
+    <meta name="apple-mobile-web-app-capable" content="yes">
+    <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
+
+    <!-- @2x -->
+    <link rel="apple-touch-icon" sizes="40x40" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-40.png') }}">
+    <link rel="apple-touch-icon" sizes="58x58" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-58.png') }}">
+    <link rel="apple-touch-icon" sizes="80x80" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-80.png') }}">
+    <link rel="apple-touch-icon" sizes="120x120" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-120.png') }}">
+    <link rel="apple-touch-icon" sizes="152x152" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-152.png') }}">
+    <link rel="apple-touch-icon" sizes="167x167" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-167.png') }}">
+    <!-- @3x -->
+    <link rel="apple-touch-icon" sizes="60x60" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-60.png') }}">
+    <link rel="apple-touch-icon" sizes="87x87" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-87.png') }}">
+    <link rel="apple-touch-icon" sizes="120x120" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-120.png') }}">
+    <link rel="apple-touch-icon" sizes="180x180" type="image/png" href="{{ url_for('static', filename='/novnc/app/images/icons/novnc-ios-180.png') }}">
+
+    <!-- Stylesheets -->
+    <link rel="stylesheet" href="{{ url_for('static', filename='/novnc/app/styles/constants.css') }}">
+    <link rel="stylesheet" href="{{ url_for('static', filename='/novnc/app/styles/base.css') }}">
+    <link rel="stylesheet" href="{{ url_for('static', filename='/novnc/app/styles/input.css') }}">
+
+    <!-- Images that will later appear via CSS -->
+    <link rel="preload" as="image" href="{{ url_for('static', filename='/novnc/app/images/info.svg') }}">
+    <link rel="preload" as="image" href="{{ url_for('static', filename='/novnc/app/images/error.svg') }}">
+    <link rel="preload" as="image" href="{{ url_for('static', filename='/novnc/app/images/warning.svg') }}">
+
+    <script type="module" crossorigin="anonymous" src="{{ url_for('static', filename='/novnc/app/error-handler.js') }}"></script>
+
+    <script type="module">
+        import UI from "{{ url_for('static', filename='/novnc/app/ui.js') }}";
+        import * as Log from "{{ url_for('static', filename='/novnc/core/util/logging.js') }}";
+
+        let response;
+
+        let defaults = {};
+        let mandatory = {};
+
+        // Default settings will be loaded from defaults.json. Mandatory
+        // settings will be loaded from mandatory.json, which the user
+        // cannot change.
+
+        try {
+            response = await fetch("{{ url_for('static', filename='/novnc/defaults.json') }}");
+            if (!response.ok) {
+                throw Error("" + response.status + " " + response.statusText);
+            }
+
+            defaults = await response.json();
+        } catch (err) {
+            Log.Error("Couldn't fetch defaults.json: " + err);
+        }
+
+        try {
+            response = await fetch("{{ url_for('static', filename='/novnc/mandatory.json') }}");
+            if (!response.ok) {
+                throw Error("" + response.status + " " + response.statusText);
+            }
+
+            mandatory = await response.json();
+        } catch (err) {
+            Log.Error("Couldn't fetch mandatory.json: " + err);
+        }
+
+        // You can also override any defaults you need here:
+        //
+        defaults['host'] = '192.168.50.124';
+        defaults['port'] = '6081';
+
+        // Or force a specific setting, preventing the user from
+        // changing it:
+        //
+        // mandatory['view_only'] = true;
+
+        // See docs/EMBEDDING.md for a list of possible settings.
+
+        UI.start({ settings: { defaults: defaults,
+                               mandatory: mandatory } });
+    </script>
+</head>
+
+<body>
+
+    <div id="noVNC_fallback_error" class="noVNC_center">
+        <div>
+            <div>noVNC encountered an error:</div>
+            <br>
+            <div id="noVNC_fallback_errormsg"></div>
+        </div>
+    </div>
+
+    <!-- noVNC control bar -->
+    <div id="noVNC_control_bar_anchor" class="noVNC_vcenter">
+
+        <div id="noVNC_control_bar">
+            <div id="noVNC_control_bar_handle" title="Hide/Show the control bar"><div></div></div>
+
+            <div class="noVNC_scroll">
+
+            <h1 class="noVNC_logo" translate="no"><span>no</span><br>VNC</h1>
+
+            <hr>
+
+            <!-- Drag/Pan the viewport -->
+            <input type="image" alt="Drag" src="{{ url_for('static', filename='/novnc/app/images/drag.svg') }}"
+                id="noVNC_view_drag_button" class="noVNC_button noVNC_hidden"
+                title="Move/Drag viewport">
+
+            <!--noVNC touch device only buttons-->
+            <div id="noVNC_mobile_buttons">
+                <input type="image" alt="Keyboard" src="{{ url_for('static', filename='/novnc/app/images/keyboard.svg') }}"
+                    id="noVNC_keyboard_button" class="noVNC_button" title="Show keyboard">
+            </div>
+
+            <!-- Extra manual keys -->
+            <input type="image" alt="Extra keys" src="{{ url_for('static', filename='/novnc/app/images/toggleextrakeys.svg') }}"
+                id="noVNC_toggle_extra_keys_button" class="noVNC_button"
+                title="Show extra keys">
+            <div class="noVNC_vcenter">
+            <div id="noVNC_modifiers" class="noVNC_panel">
+                <input type="image" alt="Ctrl" src="{{ url_for('static', filename='/novnc/app/images/ctrl.svg') }}"
+                    id="noVNC_toggle_ctrl_button" class="noVNC_button"
+                    title="Toggle Ctrl">
+                <input type="image" alt="Alt" src="{{ url_for('static', filename='/novnc/app/images/alt.svg') }}"
+                    id="noVNC_toggle_alt_button" class="noVNC_button"
+                    title="Toggle Alt">
+                <input type="image" alt="Windows" src="{{ url_for('static', filename='/novnc/app/images/windows.svg') }}"
+                    id="noVNC_toggle_windows_button" class="noVNC_button"
+                    title="Toggle Windows">
+                <input type="image" alt="Tab" src="{{ url_for('static', filename='/novnc/app/images/tab.svg') }}"
+                    id="noVNC_send_tab_button" class="noVNC_button"
+                    title="Send Tab">
+                <input type="image" alt="Esc" src="{{ url_for('static', filename='/novnc/app/images/esc.svg') }}"
+                    id="noVNC_send_esc_button" class="noVNC_button"
+                    title="Send Escape">
+                <input type="image" alt="Ctrl+Alt+Del" src="{{ url_for('static', filename='/novnc/app/images/ctrlaltdel.svg') }}"
+                    id="noVNC_send_ctrl_alt_del_button" class="noVNC_button"
+                    title="Send Ctrl-Alt-Del">
+            </div>
+            </div>
+
+            <!-- Shutdown/Reboot -->
+            <input type="image" alt="Shutdown/Reboot" src="{{ url_for('static', filename='/novnc/app/images/power.svg') }}"
+                id="noVNC_power_button" class="noVNC_button"
+                title="Shutdown/Reboot...">
+            <div class="noVNC_vcenter">
+            <div id="noVNC_power" class="noVNC_panel">
+                <div class="noVNC_heading">
+                    <img alt="" src="{{ url_for('static', filename='/novnc/app/images/power.svg') }}"> Power
+                </div>
+                <input type="button" id="noVNC_shutdown_button" value="Shutdown">
+                <input type="button" id="noVNC_reboot_button" value="Reboot">
+                <input type="button" id="noVNC_reset_button" value="Reset">
+            </div>
+            </div>
+
+            <!-- Clipboard -->
+            <input type="image" alt="Clipboard" src="{{ url_for('static', filename='/novnc/app/images/clipboard.svg') }}"
+                id="noVNC_clipboard_button" class="noVNC_button"
+                title="Clipboard">
+            <div class="noVNC_vcenter">
+            <div id="noVNC_clipboard" class="noVNC_panel">
+                <div class="noVNC_heading">
+                    <img alt="" src="{{ url_for('static', filename='/novnc/app/images/clipboard.svg') }}"> Clipboard
+                </div>
+                <p class="noVNC_subheading">
+                    Edit clipboard content in the textarea below.
+                </p>
+                <textarea id="noVNC_clipboard_text" rows=5></textarea>
+            </div>
+            </div>
+
+            <!-- Toggle fullscreen -->
+            <input type="image" alt="Full screen" src="{{ url_for('static', filename='/novnc/app/images/fullscreen.svg') }}"
+                id="noVNC_fullscreen_button" class="noVNC_button noVNC_hidden"
+                title="Full screen">
+
+            <!-- Settings -->
+            <input type="image" alt="Settings" src="{{ url_for('static', filename='/novnc/app/images/settings.svg') }}"
+                id="noVNC_settings_button" class="noVNC_button"
+                title="Settings">
+            <div class="noVNC_vcenter">
+            <div id="noVNC_settings" class="noVNC_panel">
+                <div class="noVNC_heading">
+                    <img alt="" src="{{ url_for('static', filename='/novnc/app/images/settings.svg') }}"> Settings
+                </div>
+                <ul>
+                    <li>
+                        <label>
+                            <input id="noVNC_setting_shared" type="checkbox"
+                                   class="toggle">
+                            Shared mode
+                        </label>
+                    </li>
+                    <li>
+                        <label>
+                            <input id="noVNC_setting_view_only" type="checkbox"
+                                   class="toggle">
+                            View only
+                        </label>
+                    </li>
+                    <li><hr></li>
+                    <li>
+                        <label>
+                            <input id="noVNC_setting_view_clip" type="checkbox"
+                                   class="toggle">
+                            Clip to window
+                        </label>
+                    </li>
+                    <li>
+                        <label for="noVNC_setting_resize">Scaling mode:</label>
+                        <select id="noVNC_setting_resize" name="vncResize">
+                            <option value="off">None</option>
+                            <option value="scale">Local scaling</option>
+                            <option value="remote">Remote resizing</option>
+                        </select>
+                    </li>
+                    <li><hr></li>
+                    <li>
+                        <div class="noVNC_expander">Advanced</div>
+                        <div><ul>
+                            <li>
+                                <label for="noVNC_setting_quality">Quality:</label>
+                                <input id="noVNC_setting_quality" type="range" min="0" max="9" value="6">
+                            </li>
+                            <li>
+                                <label for="noVNC_setting_compression">Compression level:</label>
+                                <input id="noVNC_setting_compression" type="range" min="0" max="9" value="2">
+                            </li>
+                            <li><hr></li>
+                            <li>
+                                <label for="noVNC_setting_repeaterID">Repeater ID:</label>
+                                <input id="noVNC_setting_repeaterID" type="text" value="">
+                            </li>
+                            <li>
+                                <div class="noVNC_expander">WebSocket</div>
+                                <div><ul>
+                                    <li>
+                                        <label>
+                                            <input id="noVNC_setting_encrypt" type="checkbox"
+                                                   class="toggle">
+                                            Encrypt
+                                        </label>
+                                    </li>
+                                    <li>
+                                        <label for="noVNC_setting_host">Host:</label>
+                                        <input id="noVNC_setting_host">
+                                    </li>
+                                    <li>
+                                        <label for="noVNC_setting_port">Port:</label>
+                                        <input id="noVNC_setting_port" type="number">
+                                    </li>
+                                    <li>
+                                        <label for="noVNC_setting_path">Path:</label>
+                                        <input id="noVNC_setting_path" type="text" value="websockify">
+                                    </li>
+                                </ul></div>
+                            </li>
+                            <li><hr></li>
+                            <li>
+                                <label>
+                                    <input id="noVNC_setting_reconnect" type="checkbox"
+                                           class="toggle">
+                                    Automatic reconnect
+                                </label>
+                            </li>
+                            <li>
+                                <label for="noVNC_setting_reconnect_delay">Reconnect delay (ms):</label>
+                                <input id="noVNC_setting_reconnect_delay" type="number">
+                            </li>
+                            <li><hr></li>
+                            <li>
+                                <label>
+                                    <input id="noVNC_setting_show_dot" type="checkbox"
+                                           class="toggle">
+                                    Show dot when no cursor
+                                </label>
+                            </li>
+                            <li><hr></li>
+                            <!-- Logging selection dropdown -->
+                            <li>
+                                <label>Logging:
+                                    <select id="noVNC_setting_logging" name="vncLogging">
+                                    </select>
+                                </label>
+                            </li>
+                        </ul></div>
+                    </li>
+                    <li class="noVNC_version_separator"><hr></li>
+                    <li class="noVNC_version_wrapper">
+                        <span>Version:</span>
+                        <span class="noVNC_version"></span>
+                    </li>
+                </ul>
+            </div>
+            </div>
+
+            <!-- Connection controls -->
+            <input type="image" alt="Disconnect" src="{{ url_for('static', filename='/novnc/app/images/disconnect.svg') }}"
+                id="noVNC_disconnect_button" class="noVNC_button"
+                title="Disconnect">
+
+            </div>
+        </div>
+
+    </div> <!-- End of noVNC_control_bar -->
+
+    <div id="noVNC_hint_anchor" class="noVNC_vcenter">
+        <div id="noVNC_control_bar_hint">
+        </div>
+    </div>
+
+    <!-- Status dialog -->
+    <div id="noVNC_status"></div>
+
+    <!-- Connect button -->
+    <div class="noVNC_center">
+        <div id="noVNC_connect_dlg">
+            <p class="noVNC_logo" translate="no"><span>no</span>VNC</p>
+            <div>
+                <button id="noVNC_connect_button">
+                    <img alt="" src="{{ url_for('static', filename='/novnc/app/images/connect.svg') }}"> Connect
+                </button>
+            </div>
+        </div>
+    </div>
+
+    <!-- Server key verification dialog -->
+    <div class="noVNC_center noVNC_connect_layer">
+    <div id="noVNC_verify_server_dlg" class="noVNC_panel"><form>
+        <div class="noVNC_heading">
+            Server identity
+        </div>
+        <div>
+            The server has provided the following identifying information:
+        </div>
+        <div id="noVNC_fingerprint_block">
+            Fingerprint:
+            <span id="noVNC_fingerprint"></span>
+        </div>
+        <div>
+            Please verify that the information is correct and press
+            "Approve". Otherwise press "Reject".
+        </div>
+        <div class="button_row">
+            <input id="noVNC_approve_server_button" type="submit" value="Approve">
+            <input id="noVNC_reject_server_button" type="button" value="Reject">
+        </div>
+    </form></div>
+    </div>
+
+    <!-- Password dialog -->
+    <div class="noVNC_center noVNC_connect_layer">
+    <div id="noVNC_credentials_dlg" class="noVNC_panel"><form>
+        <div class="noVNC_heading">
+            Credentials
+        </div>
+        <div id="noVNC_username_block">
+            <label for="noVNC_username_input">Username:</label>
+            <input id="noVNC_username_input">
+        </div>
+        <div id="noVNC_password_block">
+            <label for="noVNC_password_input">Password:</label>
+            <input id="noVNC_password_input" type="password">
+        </div>
+        <div class="button_row">
+            <input id="noVNC_credentials_button" type="submit" value="Send credentials">
+        </div>
+    </form></div>
+    </div>
+
+    <!-- Transition screens -->
+    <div id="noVNC_transition">
+        <div id="noVNC_transition_text"></div>
+        <div>
+        <input type="button" id="noVNC_cancel_reconnect_button" value="Cancel">
+        </div>
+        <div class="noVNC_spinner"></div>
+    </div>
+
+    <!-- This is where the RFB elements will attach -->
+    <div id="noVNC_container">
+        <!-- Note that Google Chrome on Android doesn't respect any of these,
+             html attributes which attempt to disable text suggestions on the
+             on-screen keyboard. Let's hope Chrome implements the ime-mode
+             style for example -->
+        <textarea id="noVNC_keyboardinput" autocapitalize="off"
+            autocomplete="off" spellcheck="false" tabindex="-1"></textarea>
+    </div>
+
+    <audio id="noVNC_bell">
+        <source src="{{ url_for('static', filename='/novnc/app/sounds/bell.oga') }}" type="audio/ogg">
+        <source src="{{ url_for('static', filename='/novnc/app/sounds/bell.mp3') }}" type="audio/mpeg">
+    </audio>
+ </body>
+</html>

bootstrap_db.py

@@ -1,50 +1,18 @@
 import uuid
 from datetime import datetime
+import logging
 from sqlalchemy.orm import sessionmaker
+from sqlalchemy.exc import SQLAlchemyError
 from app import db, app
 from app.models.models import (
     User, Universe, Project, Region, VirtualDataCenter, WorkloadHost,
-    WorkloadHostFixedResource, WorkloadHostPooledResource, Workload, Network, Image, Volume, Label, RegionAccess, NetworkPort
+    WorkloadHostFixedResource, WorkloadHostPooledResource, Workload, Image, Volume, Label, RegionAccess
 )
+from app.models.network import *
 
-# Helper function to generate IP addresses within a subnet
-def generate_ip_addresses(subnet, count):
-    """
-    Generates a list of IP addresses within a given subnet.
-    
-    Args:
-        subnet (str): The subnet in CIDR notation (e.g., "192.168.1.0/24").
-        count (int): The number of IP addresses to generate.
-    
-    Returns:
-        list: A list of IP addresses.
-    """
-    base_ip = subnet.split('/')[0]  # Extract the base IP (e.g., "192.168.1.0")
-    ip_parts = list(map(int, base_ip.split('.')))  # Split into octets
-    ip_addresses = []
-
-    for i in range(1, count + 1):  # Start from 1 to avoid the network address
-        new_ip = f"{ip_parts[0]}.{ip_parts[1]}.{ip_parts[2]}.{ip_parts[3] + i}"
-        ip_addresses.append(new_ip)
-    
-    return ip_addresses
-
-# Helper function to generate MAC addresses
-def generate_mac_addresses(count):
-    """
-    Generates a list of unique MAC addresses.
-    
-    Args:
-        count (int): The number of MAC addresses to generate.
-    
-    Returns:
-        list: A list of MAC addresses.
-    """
-    mac_addresses = []
-    for i in range(count):
-        mac = f"00:1A:2B:3C:4D:{i:02X}"  # Generate a unique MAC address
-        mac_addresses.append(mac)
-    return mac_addresses
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
 # Initialize session with application context
 with app.app_context():
@@ -53,137 +21,273 @@ with app.app_context():
     db.create_all()
 
     def create_users():
-        users = [
-            User(
-                id=uuid.uuid4(),
-                first_name=f"User {i}",
-                last_name="Doe",
-                email=f"user{i}@example.com",
-                oidc_id=str(uuid.uuid4())
-            )
-            for i in range(10)  # Create 10 users
-        ]
-        session.add_all(users)
-        session.commit()
-        return users
+        try:
+            users = [
+                User(
+                    id=uuid.uuid4(),
+                    first_name=f"User {i}",
+                    last_name="Doe",
+                    email=f"user{i}@example.com",
+                    oidc_id=str(uuid.uuid4())
+                )
+                for i in range(10)  # Create 10 users
+            ]
+            session.add_all(users)
+            session.commit()
+            logger.info(f"Created {len(users)} users.")
+            return users
+        except SQLAlchemyError as e:
+            session.rollback()
+            logger.error(f"Error creating users: {e}")
+            raise
 
     def create_universes(users):
-        universes = [
-            Universe(
-                id=uuid.uuid4(),
-                name=f"Universe {i}",
-                description=f"A test universe {i}",
-                created_by=users[i % len(users)].id  # Assign created_by to a user
-            )
-            for i in range(10)  # Create 10 universes
-        ]
-        session.add_all(universes)
-        session.commit()
-        return universes
+        try:
+            universes = [
+                Universe(
+                    id=uuid.uuid4(),
+                    name=f"Universe {i}",
+                    description=f"A test universe {i}",
+                    created_by=users[i % len(users)].id  # Assign created_by to a user
+                )
+                for i in range(10)  # Create 10 universes
+            ]
+            session.add_all(universes)
+            session.commit()
+            logger.info(f"Created {len(universes)} universes.")
+            return universes
+        except SQLAlchemyError as e:
+            session.rollback()
+            logger.error(f"Error creating universes: {e}")
+            raise
 
     def create_projects(universes):
-        projects = [
-            Project(
-                id=uuid.uuid4(),
-                name=f"Project {i}",
-                universe_id=universes[i % len(universes)].id,  # Assign to a universe
-                created_by=universes[i % len(universes)].created_by  # Assign created_by to the universe's creator
-            )
-            for i in range(10)  # Create 10 projects
-        ]
-        session.add_all(projects)
-        session.commit()
-        return projects
+        try:
+            projects = [
+                Project(
+                    id=uuid.uuid4(),
+                    name=f"Project {i}",
+                    universe_id=universes[i % len(universes)].id,  # Assign to a universe
+                    created_by=universes[i % len(universes)].created_by  # Assign created_by to the universe's creator
+                )
+                for i in range(10)  # Create 10 projects
+            ]
+            session.add_all(projects)
+            session.commit()
+            logger.info(f"Created {len(projects)} projects.")
+            return projects
+        except SQLAlchemyError as e:
+            session.rollback()
+            logger.error(f"Error creating projects: {e}")
+            raise
 
     def create_regions(users):
-        regions = [
-            Region(
-                id=uuid.uuid4(),
-                name=f"Region {i}",
-                country=f"Country {i}",
-                abbreviation=f"REG{i}",
-                created_by=users[i % len(users)].id  # Assign created_by to a user
-            )
-            for i in range(10)  # Create 10 regions
-        ]
-        session.add_all(regions)
-        session.commit()
-        return regions
+        try:
+            regions = [
+                Region(
+                    id=uuid.uuid4(),
+                    name=f"Region {i}",
+                    country=f"Country {i}",
+                    abbreviation=f"REG{i}",
+                    created_by=users[i % len(users)].id  # Assign created_by to a user
+                )
+                for i in range(10)  # Create 10 regions
+            ]
+            session.add_all(regions)
+            session.commit()
+            logger.info(f"Created {len(regions)} regions.")
+            return regions
+        except SQLAlchemyError as e:
+            session.rollback()
+            logger.error(f"Error creating regions: {e}")
+            raise
 
     def create_vdcs(projects, regions):
-        vdcs = [
-            VirtualDataCenter(
-                id=uuid.uuid4(),
-                name=f"VDC {i}",
-                project_id=projects[i % len(projects)].id,  # Assign to a project
-                region_id=regions[i % len(regions)].id,  # Assign to a region
-                created_by=projects[i % len(projects)].created_by  # Assign created_by to the project's creator
-            )
-            for i in range(10)  # Create 10 VDCs
-        ]
-        session.add_all(vdcs)
-        session.commit()
-        return vdcs
+        try:
+            vdcs = [
+                VirtualDataCenter(
+                    id=uuid.uuid4(),
+                    name=f"VDC {i}",
+                    project_id=projects[i % len(projects)].id,  # Assign to a project
+                    region_id=regions[i % len(regions)].id,  # Assign to a region
+                    created_by=projects[i % len(projects)].created_by  # Assign created_by to the project's creator
+                )
+                for i in range(10)  # Create 10 VDCs
+            ]
+            session.add_all(vdcs)
+            session.commit()
+            logger.info(f"Created {len(vdcs)} VDCs.")
+            return vdcs
+        except SQLAlchemyError as e:
+            session.rollback()
+            logger.error(f"Error creating VDCs: {e}")
+            raise
 
     def create_networks(vdcs):
-        networks = [
-            Network(
-                name=f"Network {i}",
-                vdc_id=vdcs[i % len(vdcs)].id,  # Assign to a VDC
-                vni=i + 1,  # Unique VNI for each network
-                ipv4_cidr=f"192.168.{i}.0/24",  # Assign a unique IPv4 subnet
-                ipv4_gateway=f"192.168.{i}.1"  # Assign a gateway IP
-            )
-            for i in range(10)  # Create 10 networks
-        ]
-        session.add_all(networks)
-        session.commit()
-        return networks
-
-    def create_network_ports(networks):
-        for network in networks:
-            # Generate IP and MAC addresses for ports
-            ip_addresses = generate_ip_addresses(network.ipv4_cidr, 3)  # Create 3 ports per network
-            mac_addresses = generate_mac_addresses(3)  # Generate 3 unique MAC addresses
-
-            # Create network ports
-            ports = [
-                NetworkPort(
-                    network_id=network.id,
-                    ip_address=ip_addresses[i],
-                    mac_address=mac_addresses[i],
-                    port_type="private",  # Default port type
-                    status="active"  # Default state
+        try:
+            networks = [
+                Network(
+                    name=f"Network {i}",
+                    vdc_id=vdcs[i % len(vdcs)].id,  # Assign to a VDC
+                    vni=i + 1,  # Unique VNI for each network
+                    ipv4_cidr=f"192.168.{i}.0/24",  # Assign a unique IPv4 subnet
+                    ipv4_gateway=f"192.168.{i}.1"  # Assign a gateway IP
                 )
-                for i in range(3)  # Create 3 ports per network
+                for i in range(10)  # Create 10 networks
             ]
-            session.add_all(ports)
-        session.commit()
-        return ports
+            session.add_all(networks)
+            session.commit()
+            logger.info(f"Created {len(networks)} networks.")
+            return networks
+        except SQLAlchemyError as e:
+            session.rollback()
+            logger.error(f"Error creating networks: {e}")
+            raise
+
+    def create_network_ports(networks: dict[Network]):
+        try:
+            ports = []
+            for network in networks:
+                for i in range(3):
+                    ports.append(network.create_port(session))
+            session.commit()
+            logger.info(f"Created {len(ports)} network ports.")
+            return ports
+        except SQLAlchemyError as e:
+            session.rollback()
+            logger.error(f"Error creating network ports: {e}")
+            raise
 
     def create_region_access(projects, regions):
-        region_access_entries = [
-            RegionAccess(
-                project_id=projects[i % len(projects)].id,  # Assign to a project
-                region_id=regions[i % len(regions)].id  # Assign to a region
-            )
-            for i in range(10)  # Create 10 RegionAccess entries
-        ]
-        session.add_all(region_access_entries)
-        session.commit()
-        return region_access_entries
+        try:
+            region_access_entries = [
+                RegionAccess(
+                    project_id=projects[i % len(projects)].id,  # Assign to a project
+                    region_id=regions[i % len(regions)].id  # Assign to a region
+                )
+                for i in range(10)  # Create 10 RegionAccess entries
+            ]
+            session.add_all(region_access_entries)
+            session.commit()
+            logger.info(f"Created {len(region_access_entries)} region access entries.")
+            return region_access_entries
+        except SQLAlchemyError as e:
+            session.rollback()
+            logger.error(f"Error creating region access entries: {e}")
+            raise
 
-    def bootstrap_database():
-        print("Bootstrapping database...")
-        users = create_users()
-        universes = create_universes(users)
-        projects = create_projects(universes)
-        regions = create_regions(users)
-        vdcs = create_vdcs(projects, regions)
-        networks = create_networks(vdcs)
-        network_ports = create_network_ports(networks)
-        region_access_entries = create_region_access(projects, regions)
-        print("Database bootstrapped successfully!")
+    def create_images():
+        try:
+            images = []
+            for i in range(5):  # Create 5 sample images
+                # Randomly decide if the image location is a URL or a local path
+                is_http = i % 2 == 0  # Alternate between HTTP and local paths
+                location = (
+                    f"http://example.com/images/image_{i}.img" if is_http
+                    else f"/images/image_{i}.img"
+                )
+                location_type = "HTTP" if is_http else "Local"
+
+                # Randomly assign image format (qcow2 or raw)
+                image_format = "qcow2" if i % 2 == 0 else "raw"
+
+                images.append(
+                    Image(
+                        id=uuid.uuid4(),
+                        name=f"Image {i}",
+                        description=f"A sample image {i}",
+                        location=location,
+                        location_type=location_type,
+                        size=10.0 * (i + 1),  # Size in GB
+                        os_family="Linux",
+                        os_version=f"20.0{i}",
+                        checksum=f"checksum_{i}",
+                        format=image_format,  # Add image format
+                        created_by=uuid.uuid4()  # Assign a random created_by user ID
+                    )
+                )
+
+            session.add_all(images)
+            session.commit()
+            logger.info(f"Created {len(images)} images.")
+            return images
+        except SQLAlchemyError as e:
+            session.rollback()
+            logger.error(f"Error creating images: {e}")
+            raise
+
+
+def bootstrap_database():
+    try:
+        logger.info("Bootstrapping database...")
+
+        # Check and create users if none exist
+        if session.query(User).count() == 0:
+            users = create_users()
+        else:
+            users = session.query(User).all()
+            logger.info("Users already exist, skipping creation.")
+
+        # Check and create universes if none exist
+        if session.query(Universe).count() == 0:
+            universes = create_universes(users)
+        else:
+            universes = session.query(Universe).all()
+            logger.info("Universes already exist, skipping creation.")
+
+        # Check and create projects if none exist
+        if session.query(Project).count() == 0:
+            projects = create_projects(universes)
+        else:
+            projects = session.query(Project).all()
+            logger.info("Projects already exist, skipping creation.")
+
+        # Check and create regions if none exist
+        if session.query(Region).count() == 0:
+            regions = create_regions(users)
+        else:
+            regions = session.query(Region).all()
+            logger.info("Regions already exist, skipping creation.")
+
+        # Check and create VDCs if none exist
+        if session.query(VirtualDataCenter).count() == 0:
+            vdcs = create_vdcs(projects, regions)
+        else:
+            vdcs = session.query(VirtualDataCenter).all()
+            logger.info("VDCs already exist, skipping creation.")
+
+        # Check and create networks if none exist
+        if session.query(Network).count() == 0:
+            networks = create_networks(vdcs)
+        else:
+            networks = session.query(Network).all()
+            logger.info("Networks already exist, skipping creation.")
+
+        # Check and create images if none exist
+        if session.query(Image).count() == 0:
+            images = create_images()
+        else:
+            images = session.query(Image).all()
+            logger.info("Images already exist, skipping creation.")
+
+        # Check and create network ports if none exist
+        if session.query(NetworkPort).count() == 0:
+            network_ports = create_network_ports(networks)
+        else:
+            network_ports = session.query(NetworkPort).all()
+            logger.info("Network ports already exist, skipping creation.")
+
+        # Check and create region access entries if none exist
+        if session.query(RegionAccess).count() == 0:
+            region_access_entries = create_region_access(projects, regions)
+        else:
+            region_access_entries = session.query(RegionAccess).all()
+            logger.info("Region access entries already exist, skipping creation.")
+
+        logger.info("Database bootstrapping completed successfully!")
+    except Exception as e:
+        logger.error(f"Error bootstrapping database: {e}")
+        raise
 
 if __name__ == "__main__":
     bootstrap_database()

goals.md

@@ -50,3 +50,163 @@ Spawn the VM
             Network port is created in OVS with known MAC
             VM is spawned with known name and  ID
             Response sent to server with status of VM
+
+
+VM launch flow
+User requests vm launch
+We create and add networkPort and virtual machine objects to DB
+Craft the 'create-vm' task to send to the worker accordingly
+
+---
+VM Launch payload
+{
+    "vdc": "f9640ab7-3e8f-430a-83a7-abd0a2102fa3",
+    "virtual-machines": [
+        {
+            "name": "test_vm_name",
+            "memory": 2048,
+            "vcpu": 2,
+            "volumes": [
+                {
+                    "name": "disk1", 
+                    "size_gb": 20, 
+                    "boot": true, 
+                    "type": "local", 
+                    "image":"image id or 'blank'"
+                },
+                {
+                    "name": "disk2", 
+                    "size_gb": 10, 
+                    "type": "Ceph"
+                }
+            ],
+            "networks": [
+                {
+                    "id": "{networkID}", 
+                    "mac":"00:11:22:33:44:55",
+                    "ip_address":"192.168.50.124",
+                    "open_ports":["tcp/80","udp/81","icmp","tcp/1024-2048","tcp/22:103.104.105.106"] 
+                }
+            ],
+            "tags": {
+                "tag1": "value",
+                "tag3": "value"
+            }
+            ,
+            "requirements": {
+                "tag": "value"
+            }
+            
+        }
+    ]
+}
+-----------
+Payload sent to worker from API server
+-----------
+{
+   "worker_id":"UUID(""d266b29d-a888-430d-8873-151c41fd34e3"")",
+   "task_type":"virtual-machine-create",
+   "job_details":{
+      "virtual_machine_name":"my-vm",
+      "virtual_machine_id":"UUID(""16cb0f4a-d445-48c2-ab53-b8c9821e8c68"")",
+      "desired_state":"running",
+      "virtual_machine_config":{
+         "memory":2048,
+         "vcpu":2,
+         "volumes":[
+            {
+               "id":"UUID(""e6553965-6039-44e0-96cb-e0094f3b60be"")",
+               "name":"v1",
+               "size_gb":11.0,
+               "type":"None",
+               "source":"vol.source"
+            },
+            {
+               "id":"UUID(""7309cd8e-9845-4b16-bae2-092dcb78fbea"")",
+               "name":"v2",
+               "size_gb":12.0,
+               "type":"None",
+               "source":"vol.source"
+            },
+            {
+               "id":"UUID(""79cafd2e-f29e-4a70-b6f1-2d8ae6d07d05"")",
+               "name":"v3",
+               "size_gb":13.0,
+               "type":"None",
+               "source":"vol.source"
+            }
+         ],
+         "networks":[
+            
+         ]
+      }
+   }
+}
+
+Image \ Volume flow
+
+Create a blank volume locally
+Input:
+    type: local
+    source: None
+Output:
+    type: local
+    source: None 
+    path: None
+    volume_id: {volumeID}
+
+Create a blank volume on ceph-highspeed
+Input:
+    type: ceph-highspeed
+    source: None
+Output
+    type: ceph
+    path: highspeed
+    source: None
+    volume_id: {volumeID}
+
+Create a volume locally from a URL image
+Input
+    type: local
+    source: {imageID}
+Output:
+    type: local
+    path: None
+    source: 
+        type: image
+        url: http://some-url/image.qcow2?secret=dd
+        checksum: CHECKSUM
+        format: qcow2\raw
+    volume_id: {volumeID}
+
+Create a volume on ceph-nvme from a URL image
+Input
+    type: ceph-nvme
+    source: {imageID}
+Output:
+    type: ceph
+    path: nvme
+    source: 
+        type: image
+        url: http://some-url/image.qcow2?secret=dd
+        checksum: CHECKSUM
+        format: qcow2\raw
+    volume_id: {volumeID}
+
+    
+
+Create a volume locally from a ceph-slow image
+Input
+    type: local
+    source: {imageID}
+Output:
+    type: local
+    path: None
+    source: 
+        type: image
+        url: rbd://volumes-sloq/image7
+        checksum: CHECKSUM
+        format: raw
+    volume_id: {volumeID}
+    
+Create a volume on ceph from a ceph-nvme image

libvirt_test.py.py

@@ -0,0 +1,132 @@
+import logging
+import os
+import time
+import libvirt
+import uuid 
+# from libvirt_vm_task import LibvirtVMTask  # Import the LibvirtVMTask class
+from worker_tasks.libvirt import LibvirtVMTask
+
+def test_vm_with_volume_management():
+    """
+    Test function to demonstrate VM creation with automatic volume management
+    and proper cleanup on deletion.
+    """
+    # Setup logging
+    logger = logging.getLogger("vm_test")
+    logger.setLevel(logging.INFO)
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    
+    # Create a unique VM name with timestamp
+    test_vm_name = f"test-vm-{int(time.time())}"
+    vm_id=str(uuid.uuid4())
+    logger.info(f"Starting test for VM: {vm_id}")
+    
+    try:
+        # Step 1: Define VM configuration with volume that doesn't exist yet
+        # The script will automatically create it
+        vm_config = {
+            "vm_name": test_vm_name,
+            "vm_id": vm_id,
+            "desired_state": "running",
+            "vm_config": {
+                "memory": 2048,
+                "vcpu": 2,
+                "volumes": [
+                    {"id": "disk1", "size_gb": 20, "format": "qcow2"},
+                    {"id": "disk2", "size_gb": 10, "format": "qcow2"}
+                ],
+                "networks": [
+                    {"name": "default", "mac_address": "52:54:00:12:34:56"}
+                ],
+            }
+        }
+
+
+
+        config = {
+            "default_volume_path": "/tmp"
+        }        
+        # Step 2: Create and start VM - this will also create the volume
+        logger.info("Creating and starting VM with automatic volume creation")
+        task = LibvirtVMTask(vm_config, config, logger)
+        result = task.execute()
+        
+        if result["success"]:
+            logger.info(f"VM created successfully: {result['response']['status']}")
+            
+            # Step 3: Verify VM is running
+            conn = libvirt.open("qemu:///system")
+            try:
+                domain = conn.lookupByName(vm_id)
+                if domain.isActive():
+                    logger.info("Verified: VM is running")
+                else:
+                    logger.error("VM exists but is not running")
+            except libvirt.libvirtError as e:
+                logger.error(f"Failed to find VM: {e}")
+            finally:
+                conn.close()
+            
+            
+            
+            # Step 5: Wait a moment before deleting
+            logger.info("Waiting 5 seconds before deletion...")
+            time.sleep(5)
+            
+            # Step 6: Delete VM - this will also delete the volume
+            logger.info("Deleting VM and associated volumes")
+            delete_config = {
+                "vm_id": vm_id,
+                "desired_state": "deleted"
+            }
+            delete_task = LibvirtVMTask(delete_config, config, logger)
+            delete_result = delete_task.execute()
+            
+            if delete_result["success"]:
+                logger.info(f"VM deleted successfully: {delete_result['response']['status']}")
+                
+                # Step 7: Verify VM was deleted
+                conn = libvirt.open("qemu:///system")
+                try:
+                    conn.lookupByName(vm_id)
+                    logger.error("VM still exists after deletion!")
+                except libvirt.libvirtError:
+                    logger.info("Verified: VM was successfully deleted")
+                finally:
+                    conn.close()
+                
+                
+            else:
+                logger.error(f"Failed to delete VM: {delete_result}")
+        else:
+            logger.error(f"Failed to create VM: {result}")
+    
+    except Exception as e:
+        logger.error(f"Test failed with error: {e}")
+        
+        # Emergency cleanup in case of test failure
+        try:
+            # Try to delete VM if it exists
+            conn = libvirt.open("qemu:///system")
+            try:
+                domain = conn.lookupByName(vm_id)
+                if domain.isActive():
+                    domain.destroy()
+                domain.undefine()
+                logger.info("Emergency cleanup: VM deleted")
+            except libvirt.libvirtError:
+                # VM doesn't exist, no cleanup needed
+                pass
+            finally:
+                conn.close()
+        except Exception as cleanup_error:
+            logger.error(f"Emergency cleanup failed: {cleanup_error}")
+        
+    
+    logger.info("Test completed")
+
+if __name__ == "__main__":
+    test_vm_with_volume_management()

requirements.txt

@@ -9,10 +9,4 @@ pymysql
 flask_sqlalchemy
 flask_migrate
 websocket-client
-streamlit
-aiohttp
-asyncio 
-docker
-tabulate
-psutil
-python-dotenv
+streamlit

streamlit_server.py

@@ -1,6 +1,6 @@
 import streamlit as st
 from streamlit_server.views import sidebar, universes, projects, vdcs, networks
-from streamlit_server.views import details, regions, hosts, images, containers, websocket
+from streamlit_server.views import details, regions, hosts, images, containers, websocket, virtual_machines
 from streamlit_server.utils.session import init_session_state
 
 def main():
@@ -12,11 +12,6 @@ def main():
 
     init_session_state()
 
-    # Print the values in the Streamlit app
-    print("PRE Selected Resource:", st.session_state.selected_resource)
-    print("PRE View Type:", st.session_state.view_type)
-    print("PRE Selected Resource Type:", st.session_state.selected_resource_type)
-    print("PRE Current View:", st.session_state.current_view)
 
     sidebar.render()
 
@@ -24,11 +19,7 @@ def main():
         st.warning("Please enter your API key in the sidebar to continue.")
         return
         
-    # Print the values in the Streamlit app
-    print("POST Selected Resource:", st.session_state.selected_resource)
-    print("POST View Type:", st.session_state.view_type)
-    print("POST Selected Resource Type:", st.session_state.selected_resource_type)
-    print("POST Current View:", st.session_state.current_view)
+
 
     if st.session_state.view_type == 'list':
         if st.session_state.current_view == "Universes":
@@ -55,6 +46,9 @@ def main():
         elif st.session_state.current_view == "Containers":
             print("Rendering: Containers")
             containers.render()
+        elif st.session_state.current_view == "Virtual Machines":
+            print("Rendering: Virtual Machines")
+            virtual_machines.render()
         elif st.session_state.current_view == "Websocket":
             print("Rendering: Websocket")
             websocket.render()

streamlit_server/views/containers.py

@@ -2,19 +2,31 @@ import json
 import streamlit as st
 from streamlit_server.utils.helpers import format_timestamp
 
-def render():
+
+def render_list():
     """
     Renders a list of all container workloads.
     """
     st.header("Container Workload Management")
-    
+
+    # Initialize session state variables for delete all confirmation
+    if 'container_delete_all_confirmation_shown' not in st.session_state:
+        st.session_state.container_delete_all_confirmation_shown = False
+    if 'container_delete_all_confirmed' not in st.session_state:
+        st.session_state.container_delete_all_confirmed = False
+    # Initialize session state variables for delete confirmation
+    if 'container_delete_confirmation_shown' not in st.session_state:
+        st.session_state.container_delete_confirmation_shown = False
+    if 'container_delete_confirmed' not in st.session_state:
+        st.session_state.container_delete_confirmed = False
+
     # Fetch all VDCs for the dropdown
-    vdcs = st.session_state.client.get_vdcs()
+    vdcs = st.session_state.client.get_virtual_data_centers()
     vdc_options = {vdc['name']: vdc['id'] for vdc in vdcs} if vdcs else {}
-    
+
     # Add a dropdown to select VDC
     selected_vdc_name = st.selectbox("Filter by Virtual Data Center", list(vdc_options.keys()))
-    
+
     # Fetch all Workloads_hosts for the dropdown
     workload_hosts = st.session_state.client.get_workload_hosts()
     workload_host_options = {workload_host['hostname']: workload_host['id'] for workload_host in workload_hosts} if workload_hosts else {}
@@ -42,11 +54,11 @@ def render():
                 })
                 if result:
                     st.success("Container workload created successfully!")
-    
+
     # List Container Workloads
     st.subheader("Existing Container Workloads")
     containers = st.session_state.client.get_containers()
-    
+
     if containers:
         # Filter containers based on the selected VDC
         if selected_vdc_name:
@@ -55,6 +67,40 @@ def render():
         else:
             filtered_containers = containers
         
+        # Delete All functionality
+        if not st.session_state.container_delete_all_confirmation_shown:
+            if st.button("Delete All Containers", key="delete_all_containers", type="primary", help="This will delete all visible containers"):
+                st.session_state.container_delete_all_confirmation_shown = True
+                st.rerun()
+        
+        # Show confirmation if delete was clicked
+        if st.session_state.container_delete_all_confirmation_shown and not st.session_state.container_delete_all_confirmed:
+            st.warning("Are you sure you want to delete all containers? This action cannot be undone.")
+            confirm_col1, confirm_col2 = st.columns([1, 1])
+            with confirm_col1:
+                if st.button("Yes, Delete All", key="confirm_delete_all_containers"):
+                    st.session_state.container_delete_all_confirmed = True
+                    st.rerun()
+            with confirm_col2:
+                if st.button("Cancel", key="cancel_delete_all_containers"):
+                    st.session_state.container_delete_all_confirmation_shown = False
+                    st.rerun()
+        
+        # Perform deletion if confirmed
+        if st.session_state.container_delete_all_confirmed:
+            deletion_count = 0
+            for container in filtered_containers:
+                # Call the delete API for each container
+                result = st.session_state.client.delete_container(container['id'])
+                if result:
+                    deletion_count += 1
+            
+            st.success(f"Successfully deleted {deletion_count} containers.")
+            # Reset the confirmation flags
+            st.session_state.container_delete_all_confirmation_shown = False
+            st.session_state.container_delete_all_confirmed = False
+        
+        # Display the containers
         for container in filtered_containers:
             with st.container():
                 col1, col2, col3 = st.columns([3, 1, 1])
@@ -78,14 +124,110 @@ def render():
                     st.text(f"Image: {container_name}")
                 with col2:
                     st.text(f"ID: {container['id']}")
-                with col3:
+                with col3:             
                     if st.button("View Details", key=f"view_{container['id']}"):
-                        print("Button is pressed")
                         st.session_state.selected_resource = container
                         st.session_state.view_type = 'detail'
                         st.session_state.selected_resource_type = 'container'
                         st.rerun()
+                        # st.switch_page("streamlit_server/views/containers.py&container_id=d", query_params={"team": "streamlit"})
+                        
 
     else:
         st.info("No container workloads found.")
 
+
+def render_detail():
+
+    """
+    Renders the details of a specific container workload.
+    """
+
+    container = st.session_state.client.get_container(st.session_state.selected_resource['id'])
+    if not container:
+        st.error("Container workload not found.")
+        st.session_state.selected_resource = None
+        st.session_state.view_type = 'list'
+        st.session_state.selected_resource_type = None
+        return
+
+    st.title(f"Container Workload: {container['container_id']}")
+    if st.button("Back to list view"):
+        st.session_state.view_type = 'list'
+        st.rerun()
+
+    st.header("Container Workload Details")
+    col1, col2 = st.columns(2)
+    with col1:
+        st.markdown(f"**ID:** `{container['id']}`")
+        st.markdown(f"**Created:** {format_timestamp(container.get('created_at', 'Unknown'))}")
+        st.markdown(f"**VDC ID:** {container['vdc_id']}")
+        st.markdown(f"**Workload Host ID:** {container.get('container_host_id', 'N/A')}")
+    with col2:
+        if container.get('updated_at'):
+            st.markdown(f"**Last Updated:** {format_timestamp(container['updated_at'])}")
+        st.markdown(f"**Status:** {container['status']}")
+        st.markdown(f"**Launch Parameters:** {container.get('launch_params', 'N/A')}")
+    
+    # Edit Container Workload Form
+    with st.expander("Edit Container Workload"):
+        with st.form("edit_container"):
+            new_status = st.selectbox("Status", ["running", "stopped", "pending","failed-deleted","pending-deleted","pending-allocation","failed-allocation","pending-allocated"], index=["running", "stopped", "pending","failed-deleted","pending-deleted","pending-allocation","failed-allocation","pending-allocated"].index(container['status']))
+            new_launch_params = st.text_input("Launch Parameters", value=container.get('launch_params', ''))
+            submit = st.form_submit_button("Update Container Workload")
+            
+            if submit:
+                result = st.session_state.client.edit_container(container['id'], {
+                    "status": new_status,
+                    "launch_params": new_launch_params
+                })
+                if result:
+                    st.success("Container workload updated successfully!")
+                    st.session_state.selected_resource = container
+                    st.session_state.view_type = 'detail'
+                    st.session_state.selected_resource_type = 'container'
+                    st.rerun()
+ 
+
+
+    # Delete functionality
+    if not st.session_state.container_delete_confirmation_shown:
+        if st.button("Delete container", key="delete_container", type="primary", help="This will delete visible container"):
+            st.session_state.container_delete_confirmation_shown = True
+            st.rerun()
+    
+    # Show confirmation if delete was clicked
+    if st.session_state.container_delete_confirmation_shown and not st.session_state.container_delete_confirmed:
+        st.warning("Are you sure you want to delete container? This action cannot be undone.")
+        confirm_col1, confirm_col2 = st.columns([1, 1])
+        with confirm_col1:
+            if st.button("Yes, Delete", key="confirm_delete_container"):
+                st.session_state.container_delete_confirmed = True
+                st.rerun()
+        with confirm_col2:
+            if st.button("Cancel", key="cancel_delete_container"):
+                st.session_state.container_delete_confirmation_shown = False
+                st.rerun()
+    
+    # Perform deletion if confirmed
+    if st.session_state.container_delete_confirmed:
+        result = st.session_state.client.delete_container(container['id'])
+        st.session_state.container_delete_confirmation_shown = False
+        st.session_state.container_delete_confirmed = False
+        if result:
+            st.success("Container workload deleted successfully!")
+            st.session_state.selected_resource = None
+            st.session_state.view_type = 'list'
+            st.session_state.selected_resource_type = 'container'
+            st.rerun()
+        
+        st.success(f"Successfully deleted container.")
+        # Reset the confirmation flags
+        st.session_state.container_delete_confirmation_shown = False
+        st.session_state.container_delete_confirmed = False
+
+
+if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='container':
+    render_detail()
+else:
+    render_list()

streamlit_server/views/dashboard.py

@@ -0,0 +1,34 @@
+import streamlit as st
+import pandas as pd
+from streamlit_server.utils.helpers import format_timestamp
+
+# def render():
+st.header("Dashboard")
+
+workload_hosts = st.session_state.client.get_workload_hosts()
+
+st.subheader("Workload Hosts")
+if workload_hosts:
+    workload_hosts_df = pd.DataFrame(workload_hosts)
+    st.dataframe(workload_hosts_df)
+else:
+    st.info("No workload hosts found.")
+
+images = st.session_state.client.get_images()
+
+st.subheader("Images")
+if images:
+    images_df = pd.DataFrame(images)
+    st.dataframe(images_df)
+else:
+    st.info("No workload hosts found.")
+
+
+regions = st.session_state.client.get_regions()
+
+st.subheader("regions")
+if regions:
+    regions_df = pd.DataFrame(regions)
+    st.dataframe(regions_df)
+else:
+    st.info("No workload hosts found.")

streamlit_server/views/details.py

@@ -1,208 +0,0 @@
-import streamlit as st
-import pandas as pd
-from streamlit_server.utils.helpers import format_timestamp
-
-def render():
-    resource = st.session_state.selected_resource
-    resource_type = st.session_state.selected_resource_type
-
-    if not resource:
-        st.warning("No resource selected.")
-        return
-
-    print(f"Resource type: {resource_type}")
-
-    if resource_type == 'universe':
-        render_detail_view_universe(resource)
-    elif resource_type == 'project':
-        render_detail_view_project(resource)
-    elif resource_type == 'vdc':
-        render_detail_view_vdc(resource)
-    elif resource_type == 'region':
-        render_detail_view_region(resource)
-    elif resource_type == 'network':
-        render_detail_view_network(resource)
-    elif resource_type == 'container':
-        render_detail_view_container(resource)
-
-def render_detail_view_container(container):
-
-    """
-    Renders the details of a specific container workload.
-    """
-    container = st.session_state.client.get_container(container['id'])
-    if not container:
-        st.error("Container workload not found.")
-        return
-
-    st.title(f"Container Workload: {container['container_id']}")
-
-    st.header("Container Workload Details")
-    col1, col2 = st.columns(2)
-    with col1:
-        st.markdown(f"**ID:** `{container['id']}`")
-        st.markdown(f"**Created:** {format_timestamp(container.get('created_at', 'Unknown'))}")
-        st.markdown(f"**VDC ID:** {container['vdc_id']}")
-        st.markdown(f"**Workload Host ID:** {container.get('container_host_id', 'N/A')}")
-    with col2:
-        if container.get('updated_at'):
-            st.markdown(f"**Last Updated:** {format_timestamp(container['updated_at'])}")
-        st.markdown(f"**Status:** {container['status']}")
-        st.markdown(f"**Launch Parameters:** {container.get('launch_params', 'N/A')}")
-    
-    # Edit Container Workload Form
-    with st.expander("Edit Container Workload"):
-        with st.form("edit_container"):
-            new_status = st.selectbox("Status", ["running", "stopped", "pending","failed-deleted","pending-deleted","pending-allocation","failed-allocation","pending-allocated"], index=["running", "stopped", "pending","failed-deleted","pending-deleted","pending-allocation","failed-allocation","pending-allocated"].index(container['status']))
-            new_launch_params = st.text_input("Launch Parameters", value=container.get('launch_params', ''))
-            submit = st.form_submit_button("Update Container Workload")
-            
-            if submit:
-                result = st.session_state.client.edit_container(container['id'], {
-                    "status": new_status,
-                    "launch_params": new_launch_params
-                })
-                if result:
-                    st.success("Container workload updated successfully!")
-                    st.session_state.selected_resource = container
-                    st.session_state.view_type = 'detail'
-                    st.session_state.selected_resource_type = 'container'
-                    st.rerun()
-    
-    # Delete Container Workload Button
-    if st.button("Delete Container Workload"):
-        result = st.session_state.client.delete_container(container['id'])
-        if result:
-            st.success("Container workload deleted successfully!")
-            st.session_state.selected_resource = None
-            st.session_state.view_type = 'list'
-            st.session_state.selected_resource_type = 'container'
-            st.rerun()
-
-def render_detail_view_universe(universe):
-    st.title(f"Universe: {universe['name']}")
-    
-    st.header("Universe Details")
-    col1, col2 = st.columns(2)
-    with col1:
-        st.markdown(f"**ID:** `{universe['id']}`")
-        st.markdown(f"**Created:** {format_timestamp(universe['created_at'])}")
-    with col2:
-        if universe.get('updated_at'):
-            st.markdown(f"**Last Updated:** {format_timestamp(universe['updated_at'])}")
-        if universe.get('description'):
-            st.markdown(f"**Description:** {universe['description']}")
-
-def render_detail_view_project(project):
-    st.title(f"Project: {project['name']}")
-    
-    st.header("Project Details")
-    col1, col2 = st.columns(2)
-    with col1:
-        st.markdown(f"**ID:** `{project['id']}`")
-        st.markdown(f"**Created:** {format_timestamp(project['created_at'])}")
-    with col2:
-        st.markdown(f"**Universe ID:** `{project['universe_id']}`")
-        if project.get('updated_at'):
-            st.markdown(f"**Last Updated:** {format_timestamp(project['updated_at'])}")
-
-def render_detail_view_vdc(vdc):
-    st.title(f"Virtual Data Center: {vdc['name']}")
-    
-    st.header("VDC Details")
-    col1, col2 = st.columns(2)
-    with col1:
-        st.markdown(f"**ID:** `{vdc['id']}`")
-        st.markdown(f"**Created:** {format_timestamp(vdc['created_at'])}")
-    with col2:
-        st.markdown(f"**Project ID:** `{vdc['project_id']}`")
-        if vdc.get('updated_at'):
-            st.markdown(f"**Last Updated:** {format_timestamp(vdc['updated_at'])}")
-            
-
-def render_detail_view_network(network):
-    st.title(f"Network: {network['name']}")
-    
-    st.header("Network Details")
-    col1, col2 = st.columns(2)
-    with col1:
-        st.markdown(f"**ID:** `{network['id']}`")
-        st.markdown(f"**Created:** {network.get('created_at', 'Unknown')}")
-        st.markdown(f"**IPv4 CIDR:** {network.get('ipv4_cidr', 'N/A')}")
-        st.markdown(f"**IPv4 Gateway:** {network.get('ipv4_gateway', 'N/A')}")
-        st.markdown(f"**Encapsulation Type:** {network.get('encapsulation', 'N/A')}")
-        st.markdown(f"**VNI:** {network.get('vni', 'N/A')}")
-    with col2:
-        if network.get('updated_at'):
-            st.markdown(f"**Last Updated:** {network['updated_at']}")
-        if network.get('description'):
-            st.markdown(f"**Description:** {network['description']}")
-        st.markdown(f"**IPv6 CIDR:** {network.get('ipv6_cidr', 'N/A')}")
-        st.markdown(f"**IPv6 Gateway:** {network.get('ipv6_gateway', 'N/A')}")
-    
-    st.header("Network Ports")
-    network_ports = st.session_state.client.get_network_ports_by_network(network['id'])
-    
-    if network_ports:
-        # Fetch workloads to map workload_id to workload name
-        # TODO Enable to fetching of workloads once htey exist
-        # workloads = st.session_state.client.get_workloads()
-        # workload_map = {workload['id']: workload['name'] for workload in workloads}
-        
-        # Add workload names to each network port entry
-        for port in network_ports:
-            port['workload_name'] = "Unk" #workload_map.get(port['workload_id'], 'Unknown Workload')
-        
-        # Create a DataFrame with the enriched data
-        df = pd.DataFrame(network_ports)
-        df = df[[
-            'id', 'workload_id', 'workload_name', 'ip_address', 'mac_address', 
-            'port_type', 'state'
-        ]]
-        st.dataframe(df, use_container_width=True)
-    else:
-        st.info("No network ports found for this network.")
-
-
-
-
-def render_detail_view_region(region):
-    st.title(f"Region: {region['name']}")
-    
-    st.header("Region Details")
-    col1, col2 = st.columns(2)
-    with col1:
-        st.markdown("**Region ID**")
-        st.code(f"{region['id']}")
-        st.markdown("**Region Enrollment Key**(Confidental)")
-        st.code(f"{region['enrollment_key']}")
-    with col2:
-        st.markdown(f"**Last Updated:** {region['updated_at']}")
-        st.markdown(f"**Created:** {region.get('created_at', 'Unknown')}")
-
-        if region.get('description'):
-            st.markdown(f"**Description:** {region['description']}")
-    
-    st.header("Region Access Entries")
-    region_access = st.session_state.client.get_region_access(region['id'])
-    
-    if region_access:
-        # Fetch all projects and regions
-        projects = st.session_state.client.get_projects()
-        regions = st.session_state.client.get_regions()
-        
-        # Create dictionaries to map IDs to names
-        project_map = {project['id']: project['name'] for project in projects}
-        region_map = {region['id']: region['name'] for region in regions}
-        
-        # Add project and region names to each entry
-        for entry in region_access:
-            entry['project_name'] = project_map.get(entry['project_id'], 'Unknown Project')
-            entry['region_name'] = region_map.get(entry['region_id'], 'Unknown Region')
-        
-        # Create a DataFrame with the enriched data
-        df = pd.DataFrame(region_access)
-        df = df[['project_id', 'project_name', 'region_id', 'region_name']]  # Include names in the table
-        st.dataframe(df, use_container_width=True)
-    else:
-        st.info("No access entries found for this region.")

streamlit_server/views/hosts.py

@@ -2,16 +2,16 @@ import streamlit as st
 import pandas as pd
 from streamlit_server.utils.helpers import format_timestamp
 
-def render():
-    st.header("Worker & Host Management")
-    
-    # Fetch Worker Agents & Workload Hosts
-    workload_hosts = st.session_state.client.get_workload_hosts()
-    
-    # Display Workload Hosts
-    st.subheader("Workload Hosts")
-    if workload_hosts:
-        workload_hosts_df = pd.DataFrame(workload_hosts)
-        st.dataframe(workload_hosts_df)
-    else:
-        st.info("No workload hosts found.")
+# def render():
+st.header("Worker & Host Management")
+
+# Fetch Worker Agents & Workload Hosts
+workload_hosts = st.session_state.client.get_workload_hosts()
+
+# Display Workload Hosts
+st.subheader("Workload Hosts")
+if workload_hosts:
+    workload_hosts_df = pd.DataFrame(workload_hosts)
+    st.dataframe(workload_hosts_df)
+else:
+    st.info("No workload hosts found.")

streamlit_server/views/images.py

@@ -1,32 +1,34 @@
 import streamlit as st
+import pandas as pd
 from streamlit_server.utils.helpers import format_timestamp
 
-def render_image_list():
-    """
-    Renders a list of all images.
-    """
+def render_list():
     st.header("Image Management")
     
     # Create Image Form
     with st.expander("Create New Image"):
         with st.form("create_image"):
-            location = st.text_input("Location (URL or Path)")
-            size = st.number_input("Size (in MB)", min_value=0.0)
-            location_type = st.selectbox("Location Type", ["url", "local"])
-            os_family = st.text_input("OS Family")
-            os_version = st.text_input("OS Version")
+            name = st.text_input("Image Name")
+            location = st.text_input("Image Location (e.g., /images/image.img)")
+            location_type = st.selectbox("Location Type", ["Local", "Remote"])
+            size = st.number_input("Size (GB)", min_value=0.1, step=0.1)
+            os_family = st.text_input("OS Family (e.g., Linux)")
+            os_version = st.text_input("OS Version (e.g., 20.04)")
+            checksum = st.text_input("Checksum (optional)")
             submit = st.form_submit_button("Create Image")
             
-            if submit and location and size:
-                result = st.session_state.client.create_image({
-                    "location": location,
-                    "size": size,
-                    "location_type": location_type,
-                    "os_family": os_family,
-                    "os_version": os_version
-                })
+            if submit and name and location:
+                result = st.session_state.client.create_image(
+                    name=name,
+                    location=location,
+                    location_type=location_type,
+                    size=size,
+                    os_family=os_family,
+                    os_version=os_version,
+                    checksum=checksum
+                )
                 if result:
-                    st.success(f"Image created successfully!")
+                    st.success(f"Image '{name}' created successfully!")
     
     # List Images
     st.subheader("Existing Images")
@@ -37,93 +39,45 @@ def render_image_list():
             with st.container():
                 col1, col2, col3 = st.columns([3, 1, 1])
                 with col1:
-                    st.markdown(f"### {image['location']}")
-                    st.text(f"Size: {image['size']} MB")
-                    st.text(f"OS: {image.get('os_family', 'N/A')} {image.get('os_version', 'N/A')}")
+                    st.markdown(f"### {image['name']}")
+                    st.text(f"Location: {image['location']}")
+                    st.text(f"Size: {image['size']} GB")
                 with col2:
                     st.text(f"ID: {image['id'][:8]}...")
                 with col3:
                     if st.button("View Details", key=f"view_{image['id']}"):
                         st.session_state.selected_resource = image
                         st.session_state.view_type = 'detail'
-                        st.session_state.selected_resource_type='image'
+                        st.session_state.selected_resource_type = 'image'
                         st.rerun()  # Force Streamlit to rerun the script
 
-    else:
-        st.info("No images found.")
 
-def render_image_detail(image_id):
-    """
-    Renders the details of a specific image.
-    """
-    image = st.session_state.client.get_image(image_id)
-    if not image:
-        st.error("Image not found.")
-        return
+def render_detail():
+    image = st.session_state.client.get_image(st.session_state.selected_resource['id'])
 
-    st.title(f"Image: {image['location']}")
+    st.title(f"Image: {image['name']}")
+    if st.button("Back to list view"):
+        st.session_state.view_type = 'list'
+        st.rerun()
     
     st.header("Image Details")
     col1, col2 = st.columns(2)
     with col1:
         st.markdown(f"**ID:** `{image['id']}`")
-        st.markdown(f"**Created:** {format_timestamp(image.get('created_at', 'Unknown'))}")
-        st.markdown(f"**Size:** {image['size']} MB")
-        st.markdown(f"**Location Type:** {image.get('location_type', 'N/A')}")
+        st.markdown(f"**Created:** {format_timestamp(image['created_at'])}")
+        st.markdown(f"**Location:** `{image['location']}`")
+        st.markdown(f"**Location Type:** `{image['location_type']}`")
     with col2:
+        st.markdown(f"**Size:** `{image['size']} GB`")
+        st.markdown(f"**OS Family:** `{image['os_family']}`")
+        st.markdown(f"**OS Version:** `{image['os_version']}`")
+        if image.get('checksum'):
+            st.markdown(f"**Checksum:** `{image['checksum']}`")
         if image.get('updated_at'):
             st.markdown(f"**Last Updated:** {format_timestamp(image['updated_at'])}")
-        st.markdown(f"**OS Family:** {image.get('os_family', 'N/A')}")
-        st.markdown(f"**OS Version:** {image.get('os_version', 'N/A')}")
-    
-    # Edit Image Form
-    with st.expander("Edit Image"):
-        with st.form("edit_image"):
-            new_location = st.text_input("Location (URL or Path)", value=image['location'])
-            new_size = st.number_input("Size (in MB)", min_value=0.0, value=image['size'])
-            new_location_type = st.selectbox("Location Type", ["url", "local"], index=0 if image.get('location_type') == "url" else 1)
-            new_os_family = st.text_input("OS Family", value=image.get('os_family', ''))
-            new_os_version = st.text_input("OS Version", value=image.get('os_version', ''))
-            submit = st.form_submit_button("Update Image")
-            
-            if submit:
-                result = st.session_state.client.edit_image(image['id'], {
-                    "location": new_location,
-                    "size": new_size,
-                    "location_type": new_location_type,
-                    "os_family": new_os_family,
-                    "os_version": new_os_version
-                })
-                if result:
-                    st.success("Image updated successfully!")
-                    st.session_state.selected_resource = image
-                    st.session_state.view_type = 'detail'
-                    st.session_state.selected_resource_type='image'
-    
-    # Delete Image Button
-    if st.button("Delete Image"):
-        result = st.session_state.client.delete_image(image['id'])
-        if result:
-            st.success("Image deleted successfully!")
-            st.session_state.view_type = 'list'
-            st.session_state.selected_resource_type='image'
-
-def render():
-    """
-    Main render function for the Images page.
-    """
-    # Read query parameters for routing
 
 
-    if not st.session_state.get('api_key'):
-        st.warning("Please enter your API key in the sidebar to continue.")
-        return
-
-    if st.session_state.view_type == 'list':
-        print("Viewing list")
-        render_image_list()
-    elif st.session_state.view_type == 'detail' and st.session_state.selected_resource:
-        print("Viewing detail")
-        render_image_detail(st.session_state.selected_resource["id"])
-    else:
-        st.error("Invalid view type or resource ID.")
+if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type == 'image':
+    render_detail()
+else:
+    render_list()

streamlit_server/views/networks.py

@@ -2,14 +2,14 @@ import streamlit as st
 import pandas as pd
 from streamlit_server.utils.helpers import format_timestamp
 
-def render():
+def render_list():
     st.header("Network Management")
     
     # Create Network Form
     with st.expander("Create New Network"):
         with st.form("create_network"):
             name = st.text_input("Network Name")
-            vdcs = st.session_state.client.get_vdcs()
+            vdcs = st.session_state.client.get_virtual_data_centers()
             vdc_choices = {v['name']: v['id'] for v in vdcs} if vdcs else {}
             selected_vdc = st.selectbox("Select VDC", list(vdc_choices.keys()))
             submit = st.form_submit_button("Create Network")
@@ -38,3 +38,63 @@ def render():
                         st.session_state.view_type = 'detail'
                         st.session_state.selected_resource_type='network'
                         st.rerun()  # Force Streamlit to rerun the script
+
+
+
+def render_detail():
+    network = st.session_state.client.get_network(st.session_state.selected_resource['id'])
+    if not network:
+        st.error("Network not found.")
+        st.session_state.selected_resource = None
+        st.session_state.view_type = 'list'
+        st.session_state.selected_resource_type = None
+        return
+
+    st.title(f"Network: {network['name']}")
+    if st.button("Back to list view"):
+        st.session_state.view_type = 'list'
+        st.rerun()
+
+    st.header("Network Details")
+    col1, col2 = st.columns(2)
+    with col1:
+        st.markdown(f"**ID:** `{network['id']}`")
+        st.markdown(f"**VDC ID:** `{network['vdc_id']}`")
+        st.markdown(f"**Created:** {network.get('created_at', 'Unknown')}")
+        st.markdown(f"**IPv4 CIDR:** {network.get('ipv4_cidr', 'N/A')}")
+        st.markdown(f"**IPv4 Gateway:** {network.get('ipv4_gateway', 'N/A')}")
+    with col2:
+        if network.get('updated_at'):
+            st.markdown(f"**Last Updated:** {network['updated_at']}")
+        if network.get('description'):
+            st.markdown(f"**Description:** {network['description']}")
+        st.markdown(f"**IPv6 CIDR:** {network.get('ipv6_cidr', 'N/A')}")
+        st.markdown(f"**IPv6 Gateway:** {network.get('ipv6_gateway', 'N/A')}")
+    
+    st.header("Network Ports")
+    network_ports = st.session_state.client.get_network_ports_by_network(network['id'])
+    
+    if network_ports:
+        # Fetch workloads to map workload_id to workload name
+        # TODO Enable to fetching of workloads once htey exist
+        # workloads = st.session_state.client.get_workloads()
+        # workload_map = {workload['id']: workload['name'] for workload in workloads}
+        
+        # Add workload names to each network port entry
+        for port in network_ports:
+            port['workload_name'] = "Unk" #workload_map.get(port['workload_id'], 'Unknown Workload')
+        
+        # Create a DataFrame with the enriched data
+        df = pd.DataFrame(network_ports)
+        df = df[[
+            'id', 'workload_id', 'workload_name', 'ip_address', 'mac_address', 
+            'port_type'
+        ]]
+        st.dataframe(df, use_container_width=True)
+    else:
+        st.info("No network ports found for this network.")
+
+if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='network':
+    render_detail()
+else:
+    render_list()

streamlit_server/views/projects.py

@@ -2,7 +2,7 @@ import streamlit as st
 import pandas as pd
 from streamlit_server.utils.helpers import format_timestamp
 
-def render():
+def render_list():
     st.header("Project Management")
     
     # Create Project Form
@@ -39,3 +39,27 @@ def render():
                         st.session_state.selected_resource_type='project'
                         st.rerun()  # Force Streamlit to rerun the script
 
+
+def render_detail():
+    project = st.session_state.client.get_project(st.session_state.selected_resource['id'])
+
+    st.title(f"Project: {project['name']}")
+    if st.button("Back to list view"):
+        st.session_state.view_type = 'list'
+        st.rerun()
+    
+    st.header("Project Details")
+    col1, col2 = st.columns(2)
+    with col1:
+        st.markdown(f"**ID:** `{project['id']}`")
+        st.markdown(f"**Created:** {format_timestamp(project['created_at'])}")
+    with col2:
+        st.markdown(f"**Universe ID:** `{project['universe_id']}`")
+        if project.get('updated_at'):
+            st.markdown(f"**Last Updated:** {format_timestamp(project['updated_at'])}")
+
+
+if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='project':
+    render_detail()
+else:
+    render_list()

streamlit_server/views/regions.py

@@ -2,7 +2,7 @@ import streamlit as st
 import pandas as pd
 from streamlit_server.utils.helpers import format_timestamp
 
-def render():
+def render_list():
     st.header("Region Management")
     
     # Create Region Form
@@ -39,3 +39,53 @@ def render():
                         st.session_state.selected_resource_type='region'
                         st.rerun()  # Force Streamlit to rerun the script
 
+
+
+def render_detail():
+    region = st.session_state.client.get_region(st.session_state.selected_resource['id'])
+
+    st.title(f"Region: {region['name']}")
+    if st.button("Back to list view"):
+        st.session_state.view_type = 'list'
+        st.rerun()
+
+    st.header("Region Details")
+    col1, col2 = st.columns(2)
+    with col1:
+        st.code(f"{region['id']}")
+        st.markdown(f"**Created:** {region.get('created_at', 'Unknown')}")
+    with col2:
+        if region.get('updated_at'):
+            st.markdown(f"**Last Updated:** {region['updated_at']}")
+        if region.get('description'):
+            st.markdown(f"**Description:** {region['description']}")
+    
+    st.header("Region Access Entries")
+    region_access = st.session_state.client.get_region_access(region['id'])
+    
+    if region_access:
+        # Fetch all projects and regions
+        projects = st.session_state.client.get_projects()
+        regions = st.session_state.client.get_regions()
+        
+        # Create dictionaries to map IDs to names
+        project_map = {project['id']: project['name'] for project in projects}
+        region_map = {region['id']: region['name'] for region in regions}
+        
+        # Add project and region names to each entry
+        for entry in region_access:
+            entry['project_name'] = project_map.get(entry['project_id'], 'Unknown Project')
+            entry['region_name'] = region_map.get(entry['region_id'], 'Unknown Region')
+        
+        # Create a DataFrame with the enriched data
+        df = pd.DataFrame(region_access)
+        df = df[['project_id', 'project_name', 'region_id', 'region_name']]  # Include names in the table
+        st.dataframe(df, use_container_width=True)
+    else:
+        st.info("No access entries found for this region.")
+
+
+if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='region':
+    render_detail()
+else:
+    render_list()

streamlit_server/views/sidebar.py

@@ -7,31 +7,77 @@ def render():
     if 'api_key' not in st.session_state or st.session_state.api_key == "":
         st.session_state.api_key = "xxx"
         print("Providing xxx as api key")
+    
     # Display the API key input with the default value
     api_key = st.sidebar.text_input("API Key", type="password", value=st.session_state.get('api_key'))
     if api_key:
         st.session_state.api_key = api_key
 
+    # Define views with appropriate icons
+    views = {
+        "Universes": "🌌",
+        "Projects": "📂",
+        "Virtual Data Centers": "🏢",
+        "Networks": "🔌",
+        "Regions": "🌎",
+        "Hosts": "💻",
+        "Images": "🖼️",
+        "Containers": "📦",
+        "Websocket": "🔄",
+        "Virtual Machines": "⚙️"
+    }
+    
+    # Style the sidebar navigation
+    st.sidebar.markdown("""
+    <style>
+        /* Reduce spacing between items */
+        div.row-widget.stSelectbox > div {
+            margin-bottom: 0;
+        }
+        
+        /* Style for the navbar */
+        .streamlit-expanderHeader {
+            font-size: 1em;
+            padding: 0.5rem;
+        }
+        
+        /* Make the select box more compact */
+        .stSelectbox div:first-child {
+            padding-bottom: 0;
+        }
+    </style>
+    """, unsafe_allow_html=True)
+    
     st.sidebar.header("Navigation")
-    views = ["Universes", "Projects", "Virtual Data Centers", "Networks", "Regions", "Hosts", "Images", "Containers", "Websocket"]
-
+    
+    # Initialize current view in session state if not present
+    if 'current_view' not in st.session_state:
+        st.session_state.current_view = list(views.keys())[0]
+    
+    # Format the options with icons
+    format_func = lambda x: f"{views[x]} {x}"
+    
+    # Use selectbox for navigation
     if st.session_state.view_type == 'list':
-        # if st.sidebar.radio("Select View", views):
-        #     print(f"Radio buton pressed")
-        #     st.session_state.current_view = "D"
-        for _viewType in views:
-            if st.sidebar.button(_viewType):
-                print(f"Processing sidebar button pres {_viewType}")
-                st.session_state.current_view=_viewType
-                st.session_state.selected_resource = None
-                st.session_state.view_type = 'list'
-                st.session_state.selected_resource_type = None
-                # st.rerun()  # Force Streamlit to rerun the script
+        selected_view = st.sidebar.selectbox(
+            "Select Resource Type",
+            options=list(views.keys()),
+            format_func=format_func,
+            index=list(views.keys()).index(st.session_state.current_view),
+            label_visibility="collapsed"
+        )
+        
+        # Handle navigation
+        if selected_view != st.session_state.current_view:
+            st.session_state.current_view = selected_view
+            st.session_state.selected_resource = None
+            st.session_state.view_type = 'list'
+            st.session_state.selected_resource_type = None
+            st.rerun()
     
     if st.session_state.view_type == 'detail':
         if st.sidebar.button("← Back to List View"):
             st.session_state.view_type = 'list'
             st.session_state.selected_resource = None
             st.session_state.selected_resource_type = None
-            st.rerun()  # Force Streamlit to rerun the script
-
+            st.rerun()

streamlit_server/views/universes.py

@@ -2,7 +2,7 @@ import streamlit as st
 import pandas as pd
 from streamlit_server.utils.helpers import format_timestamp
 
-def render():
+def render_list():
     st.header("Universe Management")
     
     with st.expander("Create New Universe"):
@@ -37,3 +37,33 @@ def render():
                         st.rerun()  # Force Streamlit to rerun the script
 
 
+
+def render_detail():
+    universe = st.session_state.client.get_universe(st.session_state.selected_resource['id'])
+    if not universe:
+        st.error("Universe not found.")
+        st.session_state.selected_resource = None
+        st.session_state.view_type = 'list'
+        st.session_state.selected_resource_type = None
+        return
+
+    st.title(f"Universe: {universe['name']}")
+    if st.button("Back to list view"):
+        st.session_state.view_type = 'list'
+        st.rerun()
+                
+    st.header("Universe Details")
+    col1, col2 = st.columns(2)
+    with col1:
+        st.markdown(f"**ID:** `{universe['id']}`")
+        st.markdown(f"**Created:** {format_timestamp(universe['created_at'])}")
+    with col2:
+        if universe.get('updated_at'):
+            st.markdown(f"**Last Updated:** {format_timestamp(universe['updated_at'])}")
+        if universe.get('description'):
+            st.markdown(f"**Description:** {universe['description']}")
+
+if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='universe':
+    render_detail()
+else:
+    render_list()            

streamlit_server/views/virtual_data_centers.py

@@ -2,13 +2,13 @@ import streamlit as st
 import pandas as pd
 from streamlit_server.utils.helpers import format_timestamp
 
-def render():
-    st.header("Virtual Data Center (VDC) Management")
+def render_list():
+    st.header("Virtual Data Center Management")
     
     # Create VDC Form
-    with st.expander("Create New VDC"):
+    with st.expander("Create New Virtual Data Center"):
         with st.form("create_vdc"):
-            name = st.text_input("VDC Name")
+            name = st.text_input("Virtual Data Center Name")
             projects = st.session_state.client.get_projects()
             project_choices = {p['name']: p['id'] for p in projects} if projects else {}
             selected_project = st.selectbox("Select Project", list(project_choices.keys()))
@@ -17,11 +17,11 @@ def render():
             if submit and name and selected_project:
                 result = st.session_state.client.create_vdc(name, project_choices[selected_project])
                 if result:
-                    st.success(f"VDC '{name}' created successfully!")
+                    st.success(f"Virtual Data Center '{name}' created successfully!")
     
     # List VDCs
-    st.subheader("Existing VDCs")
-    vdcs = st.session_state.client.get_vdcs()
+    st.subheader("Existing Virtual Data Center")
+    vdcs = st.session_state.client.get_virtual_data_centers()
     # projects = st.session_state.client.get_projects()
     # regions = st.session_state.client.get_regions()
     
@@ -41,7 +41,31 @@ def render():
                     if st.button("View Details", key=f"view_{vdc['id']}"):
                         st.session_state.selected_resource = vdc
                         st.session_state.view_type = 'detail'
-                        st.session_state.selected_resource_type='vdc'
+                        st.session_state.selected_resource_type='virtual_data_center'
                         st.rerun()  # Force Streamlit to rerun the script
 
 
+
+def render_detail():
+    
+    vdc = st.session_state.client.get_virtual_data_center(st.session_state.selected_resource['id'])
+    st.title(f"Virtual Data Center: {vdc['name']}")
+
+    if st.button("Back to list view"):
+        st.session_state.view_type = 'list'
+        st.rerun()
+
+    st.header("Virtual Data Center Details")
+    col1, col2 = st.columns(2)
+    with col1:
+        st.markdown(f"**ID:** `{vdc['id']}`")
+        st.markdown(f"**Created:** {format_timestamp(vdc['created_at'])}")
+    with col2:
+        st.markdown(f"**Project ID:** `{vdc['project_id']}`")
+        if vdc.get('updated_at'):
+            st.markdown(f"**Last Updated:** {format_timestamp(vdc['updated_at'])}")
+
+if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='virtual_data_center':
+    render_detail()
+else:
+    render_list()            

streamlit_server/views/virtual_machines.py

@@ -0,0 +1,288 @@
+import json
+import streamlit as st
+from streamlit_server.utils.helpers import format_timestamp
+
+
+def render_list():
+    """
+    Renders a list of all VM workloads.
+    """
+    st.header("VM Workload Management")
+    
+    # Initialize session state variables for delete confirmation
+    if 'delete_confirmation_shown' not in st.session_state:
+        st.session_state.delete_confirmation_shown = False
+    if 'delete_confirmed' not in st.session_state:
+        st.session_state.delete_confirmed = False
+    
+    # Fetch all VDCs for the dropdown
+    vdcs = st.session_state.client.get_virtual_data_centers()
+    vdc_options = {vdc['name']: vdc['id'] for vdc in vdcs} if vdcs else {}
+    
+    # Add a dropdown to select VDC
+    selected_vdc_name = st.selectbox("Filter by Virtual Data Center", list(vdc_options.keys()))
+    
+    # Fetch all Workload Hosts for the dropdown
+    workload_hosts = st.session_state.client.get_workload_hosts()
+    workload_host_options = {workload_host['hostname']: workload_host['id'] for workload_host in workload_hosts} if workload_hosts else {}
+
+    # Create VM Workload Form
+    with st.expander("Create New VM Workload"):
+        with st.form("create_virtual_machine"):
+            # Dropdown for VDC selection
+            selected_vdc_name_create = st.selectbox("VDC", list(vdc_options.keys()))
+            vm_name = st.text_input("VM Name", value="my-vm")
+            memory = st.number_input("Memory (MB)", min_value=512, value=2048)
+            vcpu = st.number_input("vCPU", min_value=1, value=2)
+            
+            # Networks Dropdown
+            networks = st.session_state.client.get_networks()
+            network_options = {net['id']: net['name'] for net in networks}
+            selected_network_ids = st.multiselect("Networks", options=list(network_options.keys()), format_func=lambda x: network_options[x])
+            
+            # Volumes Section
+            st.subheader("Volumes")
+            if 'volumes' not in st.session_state:
+                st.session_state.volumes = []
+
+            for i, volume in enumerate(st.session_state.volumes):
+                col1, col2, col3, col4 = st.columns([3, 2, 2, 2])
+                
+                with col1:
+                    volume_name = st.text_input(f"Volume Name {i+1}", value=volume['name'], key=f"volume_name_{i}")
+                
+                with col2:
+                    volume_size = st.number_input(f"Size (GB) {i+1}", min_value=1, value=volume['size_gb'], key=f"volume_size_{i}")
+                
+                with col3:
+                    volume_type = st.selectbox(f"Type {i+1}", options=["local", "ceph"], index=0 if volume.get('type', 'local') == 'local' else 1, key=f"volume_type_{i}")
+                                
+                with col4:
+                    # Get image options
+                    image_list = st.session_state.client.get_images()
+
+                    # Create a mapping of image names to IDs
+                    image_name_to_id = {img['name']: img['id'] for img in image_list}
+                    image_name_to_id["blank"] = ""  # Add a blank option
+
+                    # Default selected image name (fallback to 'blank' if not found)
+                    default_image_name = volume.get('image_name', 'blank')
+                    
+                    # Dropdown to display image names
+                    selected_image_name = st.selectbox(
+                        f"Image {i+1}",
+                        options=list(image_name_to_id.keys()),
+                        index=list(image_name_to_id.keys()).index(default_image_name) if default_image_name in image_name_to_id else 0,
+                        key=f"volume_image_{i}"
+                    )
+
+                    # Get the corresponding image ID for the selected image name
+                    selected_image_id = image_name_to_id[selected_image_name]
+
+                # Store updated volume info
+                if selected_image_name == "blank":
+                    st.session_state.volumes[i] = {"name": volume_name, "size_gb": volume_size, "type": volume_type}
+                else:
+                    st.session_state.volumes[i] = {"name": volume_name, "size_gb": volume_size, "type": volume_type, "image_id": selected_image_id}
+
+            submit = st.form_submit_button("Create VM Workload")
+        
+        # **Move the buttons OUTSIDE of the form**
+        col1, col2 = st.columns([1, 1])
+        with col1:
+            if st.button("Add Volume"):
+                st.session_state.volumes.append({"name": "", "size_gb": 10, "type": "", "image":""})
+                st.rerun()
+
+        with col2:
+            for i in range(len(st.session_state.volumes)):
+                if st.button(f"Remove Volume {i+1}", key=f"remove_volume_{i}"):
+                    st.session_state.volumes.pop(i)
+                    st.rerun()
+
+        if submit and selected_vdc_name_create and vm_name:
+            # Use the selected VDC's ID
+            vdc_id = vdc_options[selected_vdc_name_create]
+
+            result = st.session_state.client.create_virtual_machine({
+                "virtual_data_center": vdc_id,
+                "virtual-machines": [{
+                    "name": vm_name,
+                    "memory": memory,
+                    "vcpu": vcpu,
+                    "volumes": st.session_state.volumes,
+                    "networks": [{"id": net_id} for net_id in selected_network_ids]
+                }]
+            })
+            if result:
+                st.success("VM workload created successfully!")
+                st.session_state.volumes = []  # Reset volumes after creation
+                st.rerun()
+
+    # List Container Workloads
+    st.subheader("Existing Container Workloads")
+    VirtualMachines = st.session_state.client.get_virtual_machines()
+    print(VirtualMachines)
+    if VirtualMachines:
+        # Filter VirtualMachines based on the selected VDC
+        if selected_vdc_name:
+            selected_vdc_id = vdc_options[selected_vdc_name]
+            filtered_VirtualMachines = [VirtualMachine for VirtualMachine in VirtualMachines if VirtualMachine['vdc_id'] == selected_vdc_id]
+        else:
+            filtered_VirtualMachines = VirtualMachines
+        
+        # Delete All functionality
+        if not st.session_state.delete_confirmation_shown:
+            if st.button("Delete All VMs", key="delete_all_vms", type="primary", help="This will delete all visible virtual machines"):
+                st.session_state.delete_confirmation_shown = True
+                st.rerun()
+        
+        # Show confirmation if delete was clicked
+        if st.session_state.delete_confirmation_shown and not st.session_state.delete_confirmed:
+            st.warning("Are you sure you want to delete all virtual machines? This action cannot be undone.")
+            confirm_col1, confirm_col2 = st.columns([1, 1])
+            with confirm_col1:
+                if st.button("Yes, Delete All", key="confirm_delete_all"):
+                    st.session_state.delete_confirmed = True
+                    st.rerun()
+            with confirm_col2:
+                if st.button("Cancel", key="cancel_delete_all"):
+                    st.session_state.delete_confirmation_shown = False
+                    st.rerun()
+        
+        # Perform deletion if confirmed
+        if st.session_state.delete_confirmed:
+            deletion_count = 0
+            for vm in filtered_VirtualMachines:
+                # Call the delete API for each VM
+                result = st.session_state.client.delete_virtual_machine(vm['id'])
+                if result:
+                    deletion_count += 1
+            
+            st.success(f"Successfully deleted {deletion_count} virtual machines.")
+            # Reset the confirmation flags
+            st.session_state.delete_confirmation_shown = False
+            st.session_state.delete_confirmed = False
+            filtered_VirtualMachines=[]
+    
+        # Display the VMs
+        for VirtualMachine in filtered_VirtualMachines:
+            with st.container():
+                col1, col2, col3 = st.columns([3, 1, 1])
+                with col1:
+                    # Safely handle empty or None launch_params
+                    launch_params = VirtualMachine.get('launch_params')
+                    if launch_params:
+                        try:
+                            VirtualMachine_name = json.loads(launch_params).get('docker_image')
+                        except json.JSONDecodeError:
+                            print("LP not valid json")
+                            VirtualMachine_name = None
+                    else:
+                        print("lp empty")
+                        VirtualMachine_name = None
+                    st.markdown(f"### {VirtualMachine['name']}")
+                    
+                    # Create a grey box for the status
+                    status = VirtualMachine['status']
+                    if status == "error":
+                        status_color = "red"
+                    elif status == "running":
+                        status_color = "green"
+                    elif status == "pending-allocated":
+                        status_color = "orange"
+                    else:
+                        status_color = "blue"
+                    
+                    st.markdown(
+                        f"""
+                        <div style="background-color: #f0f0f0; padding: 10px; border-radius: 5px;">
+                            <strong>Status:</strong> <span style="color: {status_color}; font-weight: bold;">{status}</span>
+                        </div>
+                        """,
+                        unsafe_allow_html=True
+                    )
+                    
+                    st.text(f"VDC ID: {VirtualMachine['vdc_id']}")
+                    st.text(f"Image: {VirtualMachine_name}")
+                with col2:
+                    st.text(f"ID: {VirtualMachine['id']}")
+                with col3:
+                    if st.button("View Details", key=f"view_{VirtualMachine['id']}"):
+                        print("Button is pressed")
+                        st.session_state.selected_resource = VirtualMachine
+                        st.session_state.view_type = 'detail'
+                        st.session_state.selected_resource_type = 'VirtualMachine'
+                        st.rerun()
+
+    else:
+        st.info("No VirtualMachine workloads found.")
+
+
+
+
+
+def render_detail():
+    """
+    Renders the details of a specific VM workload.
+    """
+    VirtualMachine = st.session_state.client.get_virtual_machine(st.session_state.selected_resource['id'])
+    if not VirtualMachine:
+        st.error("VM workload not found.")
+        return
+
+    st.title(f"VM Workload: {VirtualMachine['name']}")
+    if st.button("Back to list view"):
+        st.session_state.view_type = 'list'
+        st.rerun()
+
+    st.header("VM Workload Details")
+    col1, col2 = st.columns(2)
+    with col1:
+        st.markdown(f"**ID:** `{VirtualMachine['id']}`")
+        st.markdown(f"**Created:** {format_timestamp(VirtualMachine.get('created_at', 'Unknown'))}")
+        st.markdown(f"**VDC ID:** {VirtualMachine['vdc_id']}")
+        st.markdown(f"**Workload Host ID:** {VirtualMachine.get('workload_host_id', 'N/A')}")
+    with col2:
+        if VirtualMachine.get('updated_at'):
+            st.markdown(f"**Last Updated:** {format_timestamp(VirtualMachine['updated_at'])}")
+        st.markdown(f"**Status:** {VirtualMachine['status']}")
+        st.markdown(f"**Launch Parameters:** {VirtualMachine.get('launch_params', 'N/A')}")
+    
+    # Edit VM Workload Form
+    with st.expander("Edit VM Workload"):
+        with st.form("edit_vm"):
+            new_status = st.selectbox("Status", ["running", "stopped", "pending", "failed-deleted", "pending-deleted", "pending-allocation", "failed-allocation", "pending-allocated"], index=["running", "stopped", "pending", "failed-deleted", "pending-deleted", "pending-allocation", "failed-allocation", "pending-allocated"].index(VirtualMachine['status']))
+            new_launch_params = st.text_input("Launch Parameters", value=VirtualMachine.get('launch_params', ''))
+            submit = st.form_submit_button("Update VM Workload")
+            
+            if submit:
+                result = st.session_state.client.edit_virtual_machine(VirtualMachine['id'], {
+                    "status": new_status,
+                    "launch_params": new_launch_params
+                })
+                if result:
+                    st.success("VM workload updated successfully!")
+                    st.session_state.selected_resource = vm
+                    st.session_state.view_type = 'detail'
+                    st.session_state.selected_resource_type = 'VirtualMachine'
+                    st.rerun()
+    
+    # Delete VM Workload Button
+    if st.button("Delete VM Workload"):
+        result = st.session_state.client.delete_virtual_machine(VirtualMachine['id'])
+        if result:
+            st.success("VM workload deleted successfully!")
+            st.session_state.selected_resource = None
+            st.session_state.view_type = 'list'
+            st.session_state.selected_resource_type = 'VirtualMachine'
+            st.rerun()
+
+
+
+
+
+if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='VirtualMachine':
+    render_detail()
+else:
+    render_list()  

streamlit_server/views/websocket.py

@@ -35,3 +35,5 @@ def render():
                     st.error("Message cannot be empty!")
     else:
         st.warning("No connected clients found.")
+
+render()

streamlit_server/views/workload_hosts.py

@@ -0,0 +1,201 @@
+import streamlit as st
+from streamlit_server.utils.helpers import format_timestamp
+import uuid
+
+def render_list():
+    st.header("Workload Host Management")
+    
+    # Create Workload Host Form
+    with st.expander("Create New Workload Host"):
+        with st.form("create_workload_host"):
+            name = st.text_input("Workload Host Name")
+            region_id = st.text_input("Region ID")
+            secret_key = st.text_input("Secret Key", value=str(uuid.uuid4()))
+            system_manufacturer = st.text_input("System Manufacturer")
+            system_model = st.text_input("System Model")
+            physical_identifier = st.text_input("Physical Identifier")
+            dcim_identifier = st.text_input("DCIM Identifier")
+            installed_date = st.date_input("Installed Date")
+            installed_status = st.text_input("Installed Status")
+            hostname = st.text_input("Hostname")
+            available_for_scheduling = st.checkbox("Available for Scheduling", value=True)
+            placement_priority = st.number_input("Placement Priority", value=100)
+            submit = st.form_submit_button("Create Workload Host")
+            
+            if submit and name and region_id:
+                result = st.session_state.client.create_workload_host(
+                    name=name,
+                    region_id=region_id,
+                    secret_key=secret_key,
+                    system_manufacturer=system_manufacturer,
+                    system_model=system_model,
+                    physical_identifier=physical_identifier,
+                    dcim_identifier=dcim_identifier,
+                    installed_date=installed_date,
+                    installed_status=installed_status,
+                    hostname=hostname,
+                    available_for_scheduling=available_for_scheduling,
+                    placement_priority=placement_priority
+                )
+                if result:
+                    st.success(f"Workload Host '{name}' created successfully!")
+    
+    # List Workload Hosts
+    st.subheader("Existing Workload Hosts")
+    workload_hosts = st.session_state.client.get_workload_hosts()
+    
+    if workload_hosts:
+        for host in workload_hosts:
+            with st.container():
+                col1, col2, col3 = st.columns([3, 1, 1])
+                with col1:
+                    st.markdown(f"### {host['name']}")
+                    st.text(f"Hostname: {host['hostname']}")
+                    st.text(f"Region ID: {host['region_id']}")
+                with col2:
+                    st.text(f"ID: {host['id'][:8]}...")
+                with col3:
+                    if st.button("View Details", key=f"view_{host['id']}"):
+                        st.session_state.selected_resource = host
+                        st.session_state.view_type = 'detail'
+                        st.session_state.selected_resource_type = 'workload_host'
+                        st.rerun()  # Force Streamlit to rerun the script
+
+def render_detail():
+    host_id = st.session_state.selected_resource['id']
+    workload_host = st.session_state.client.get_workload_host(host_id)
+
+    st.title(f"Workload Host: {workload_host['name']}")
+    if st.button("Back to list view"):
+        st.session_state.view_type = 'list'
+        st.rerun()
+    
+    st.header("Workload Host Details")
+    col1, col2 = st.columns(2)
+    with col1:
+        st.markdown(f"**ID:** `{workload_host['id']}`")
+        st.markdown(f"**Hostname:** `{workload_host['hostname']}`")
+        # Make Region ID clickable
+        if st.button(f"**Region ID:** `{workload_host['region_id']}`"):
+            st.session_state.selected_resource = {"id": workload_host['region_id']}
+            st.session_state.view_type = 'detail'
+            st.session_state.selected_resource_type = 'region'
+            st.switch_page("streamlit_server/views/regions.py")
+            
+        st.markdown(f"**System Manufacturer:** `{workload_host['system_manufacturer']}`")
+        st.markdown(f"**System Model:** `{workload_host['system_model']}`")
+    with col2:
+        st.markdown(f"**Physical Identifier:** `{workload_host['physical_identifier']}`")
+        st.markdown(f"**DCIM Identifier:** `{workload_host['dcim_identifier']}`")
+        st.markdown(f"**Installed Status:** `{workload_host['installed_status']}`")
+        st.markdown(f"**Available for Scheduling:** `{workload_host['available_for_scheduling']}`")
+        st.markdown(f"**Placement Priority:** `{workload_host['placement_priority']}`")
+    
+    # Display OVS Bridges
+    st.subheader("Attached OVS Bridges")
+    ovs_bridges = st.session_state.client.get_ovs_bridges_for_host(host_id)
+    if ovs_bridges:
+        for bridge in ovs_bridges:
+            st.markdown(f"- **{bridge['ovs_bridge_name']}**")
+    else:
+        st.info("No OVS bridges attached to this host.")
+
+     # Display Workloads
+    st.subheader("Assigned Workloads")
+    workloads = st.session_state.client.get_workloads_for_host(host_id)
+    
+    if workloads:
+        containers = [w for w in workloads if w['workload_type'] == 'container']
+        vms = [w for w in workloads if w['workload_type'] == 'virtual_machine']
+
+        if containers:
+            st.markdown("### Containers")
+            # Table header
+            header_cols = st.columns([3, 2, 2, 2, 2])
+            header_cols[0].markdown("**Name**")
+            header_cols[1].markdown("**ID**")
+            header_cols[2].markdown("**Status**")
+            header_cols[3].markdown("**Created At**")
+            header_cols[4].markdown("**Actions**")
+
+            # Container rows
+            for container in containers:
+                cols = st.columns([3, 2, 2, 2, 2])
+                cols[0].markdown(container['name'])
+                cols[1].markdown(f"`{container['id'][:8]}...`")
+                cols[2].markdown(f"`{container['status']}`")
+                cols[3].markdown(format_timestamp(container['created_at']))
+                
+                with cols[4]:
+                    with st.popover("⚙️"):
+                        if st.button("View Details", key=f"view_container_{container['id']}"):
+                            st.session_state.selected_resource = container
+                            st.session_state.view_type = 'detail'
+                            st.session_state.selected_resource_type = 'container'
+                            st.switch_page("streamlit_server/views/containers.py")
+                        if st.button("Delete", key=f"delete_container_{container['id']}"):
+                            st.session_state.client.delete_container(container['id'])
+                            st.rerun()
+                        if st.button("Migrate", key=f"migrate_container_{container['id']}"):
+                            st.session_state.client.migrate_workload(container['id'], "NEW_HOST_ID")
+                            st.rerun()
+
+        if vms:
+            st.markdown("### Virtual Machines")
+            # Table header
+            header_cols = st.columns([3, 2, 2, 2, 2])
+            header_cols[0].markdown("**Name**")
+            header_cols[1].markdown("**ID**")
+            header_cols[2].markdown("**Status**")
+            header_cols[3].markdown("**Created At**")
+            header_cols[4].markdown("**Actions**")
+
+            # VM rows
+            for vm in vms:
+                cols = st.columns([3, 2, 2, 2, 2])
+                cols[0].markdown(vm['name'])
+                cols[1].markdown(f"`{vm['id'][:8]}...`")
+                cols[2].markdown(f"`{vm['status']}`")
+                cols[3].markdown(format_timestamp(vm['created_at']))
+                
+                with cols[4]:
+                    with st.popover("⚙️"):
+                        if st.button("View Details", key=f"view_vm_{vm['id']}"):
+                            st.session_state.selected_resource = vm
+                            st.session_state.view_type = 'detail'
+                            st.session_state.selected_resource_type = 'vm'
+                            st.switch_page("streamlit_server/views/virtual_machines.py")
+                        if st.button("Delete", key=f"delete_vm_{vm['id']}"):
+                            st.session_state.client.delete_virtual_machine(vm['id'])
+                            st.rerun()
+                        if st.button("Migrate", key=f"migrate_vm_{vm['id']}"):
+                            st.session_state.client.migrate_workload(vm['id'], "NEW_HOST_ID")
+                            st.rerun()
+
+        else:
+            st.info("No virtual machines assigned to this host.")
+    else:
+        st.info("No workloads assigned to this host.")
+
+    # Display Fixed Resources
+    st.subheader("Fixed Resources")
+    fixed_resources = st.session_state.client.get_fixed_resources_for_host(host_id)
+    if fixed_resources:
+        for resource in fixed_resources:
+            st.markdown(f"- **{resource['type']}** (Model: `{resource['model']}`, Manufacturer: `{resource['manufacturer']}`)")
+    else:
+        st.info("No fixed resources attached to this host.")
+
+    # Display Pooled Resources
+    st.subheader("Pooled Resources")
+    pooled_resources = st.session_state.client.get_pooled_resources_for_host(host_id)
+    if pooled_resources:
+        for resource in pooled_resources:
+            st.markdown(f"- **{resource['resource_type']}** (Total: `{resource['total_quantity']}`, In Use: `{resource['quantity_in_use']}`, Available: `{resource['quantity_available']}`)")
+    else:
+        st.info("No pooled resources attached to this host.")
+
+if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type == 'workload_host':
+    render_detail()
+else:
+    render_list()

streamlit_server2.py

@@ -0,0 +1,36 @@
+import streamlit as st
+from streamlit_server.utils.session import init_session_state
+
+# Define pages with icons
+pages = {
+    "Your account": [
+        st.Page("streamlit_server/views/dashboard.py", title="📊 Dashboard"),
+        st.Page("streamlit_server/views/hosts.py", title="🖥️ Hosts"),
+        st.Page("streamlit_server/views/images.py", title="🖼️ Images"),
+        st.Page("streamlit_server/views/networks.py", title="🌐 Networks"),
+        st.Page("streamlit_server/views/projects.py", title="📂 Projects"),
+        st.Page("streamlit_server/views/universes.py", title="🌌 Universes"),
+        st.Page("streamlit_server/views/regions.py", title="🗺️ Regions"),
+        st.Page("streamlit_server/views/virtual_data_centers.py", title="🏢 Virtual Data Centers"),
+        st.Page("streamlit_server/views/websocket.py", title="🔌 Active websockets"),
+    ],
+    "Workloads": [
+        st.Page("streamlit_server/views/workload_hosts.py", title="🖥️ Workload Hosts"),
+        st.Page("streamlit_server/views/containers.py", title="📦 Containers"),
+        st.Page("streamlit_server/views/virtual_machines.py", title="💻 Virtual Machines"),
+    ],
+}
+
+# Set page config
+st.set_page_config(
+    page_title="IaaS Management Portal",
+    page_icon="🌐",
+    layout="wide"
+)
+
+# Initialize session state
+init_session_state()
+
+# Run navigation
+pg = st.navigation(pages)
+pg.run()

websocket_server.py

@@ -474,20 +474,131 @@ def handle_send_message(data):
     except Exception as e:
         logger.error(f"Error sending message: {e}")
 
+# @socketio.on("docker_event")
+# def handle_docker_event(data):
+#     # TODO - Somehow we need to validate the authenticity of the worker ID, can we crosscheck the workerID suppied in the payload vs the socket id?
+#     event_type = data["event_type"]
+#     container_id = data["container_id"]
+#     container_name = data["container_name"]
+#     worker_id = data["worker_id"]
+#     timestamp = data[""]
+#     attributes = data["attributes"]
+    
+#     # TODO - Figure out what has happened and make an API request back tot he API server updating the releveant resource
+#     # Pay attention to container deleted events
+#     # Make sure the API server cross checks the worker ID this event is coming from and the resource ID it's updating
+    
+
+@socketio.on("libvirt_event")
+def handle_libvirt_event(data):
+    """
+    Handle libvirt events received from the worker.
+    Extract relevant data and log it to the console.
+    """
+    try:
+        # Extract relevant data from the event
+        worker_id = data.get("worker_id")
+        event_type = data.get("type")
+        details = data.get("details", {})
+
+        libvirt_vm_id = details.get("libvirt_vm_id")
+        system_vm_id = details.get("system_vm_id")
+        unix_timestamp = details.get("timestamp")
+        timestamp=datetime.utcfromtimestamp(unix_timestamp).isoformat()
+
+
+        # Log the extracted data
+        logger.info(f"Received libvirt event from worker {worker_id}:")
+        logger.info(f"  Event Type: {event_type}")
+        logger.info(f"  Libvirt VM ID: {libvirt_vm_id}")
+        logger.info(f"  System VM ID: {system_vm_id}")
+        logger.info(f"  Timestamp: {timestamp}")
+
+        # Optionally, log the full details for debugging purposes
+        logger.debug(f"Full event details: {data}")
+
+        if event_type=="libvirt_started":
+            #Send the update to the API Server
+            # system_vm_id
+            # worker_id
+            # state="started"
+            # timestamp
+            URL=f"/workloads/virtual_machines/status_update/{system_vm_id}"
+            logger.debug(f"Sending task update payload to API server for VM ID {system_vm_id}")
+
+            payload = {
+                "new_status": "running",
+                "system_vm_id": system_vm_id,
+                "worker_id": worker_id,
+                "timestamp": timestamp
+            }
+
+            headers = {"Content-Type": "application/json"}
+            websocket_server_response = requests.put(f"{api_server_url}/workloads/virtual_machines/status_update/{system_vm_id}", data=json.dumps(payload), headers=headers)
+            logger.debug( websocket_server_response)
+            logger.info("Update complete")
+
+        if event_type=="libvirt_stopped":
+            #Send the update to the API Server
+            # system_vm_id
+            # worker_id
+            # state="started"
+            # timestamp
+            URL=f"/workloads/virtual_machines/status_update/{system_vm_id}"
+            logger.debug(f"Sending task update payload to API server for VM ID {system_vm_id}")
+
+            payload = {
+                "new_status": "stopped",
+                "system_vm_id": system_vm_id,
+                "worker_id": worker_id,
+                "timestamp": timestamp
+            }
+
+            headers = {"Content-Type": "application/json"}
+            websocket_server_response = requests.put(f"{api_server_url}/workloads/virtual_machines/status_update/{system_vm_id}", data=json.dumps(payload), headers=headers)
+            logger.debug( websocket_server_response)
+            logger.info("Update complete")
+
+
+    except Exception as e:
+        logger.error(f"Error processing libvirt event: {e}")
+
 @socketio.on("docker_event")
 def handle_docker_event(data):
-    # TODO - Somehow we need to validate the authenticity of the worker ID, can we crosscheck the workerID suppied in the payload vs the socket id?
-    event_type = data["event_type"],
-    container_id = data["container_id"],
-    container_name = data["container_name"],
-    worker_id = data["worker_id"],
-    timestamp = data[""]
-    attributes = data["attributes"],
-    
-    # TODO - Figure out what has happened and make an API request back tot he API server updating the releveant resource
-    # Pay attention to container deleted events
-    # Make sure the API server cross checks the worker ID this event is coming from and the resource ID it's updating
-    
+    """
+    Handle Docker events received from the worker.
+    Extract relevant data and log it to the console.
+    """
+    try:
+        # Extract relevant data from the event
+        worker_id = data.get("worker_id")
+        event_type = data.get("type")
+        details = data.get("details", {})
+
+        container_id = details.get("container_id")
+        system_container_id = details.get("system_container_id")
+        container_name = details.get("container_name")
+        timestamp = details.get("timestamp")
+        status = details.get("details", {}).get("status")
+        image = details.get("details", {}).get("from")
+        action = details.get("details", {}).get("Action")
+
+        # Log the extracted data
+        logger.info(f"Received Docker event from worker {worker_id}:")
+        logger.info(f"  Event Type: {event_type}")
+        logger.info(f"  Docker Container ID: {container_id}")
+        logger.info(f"  System Container ID: {system_container_id}")
+        logger.info(f"  Container Name: {container_name}")
+        logger.info(f"  Timestamp: {timestamp}")
+        logger.info(f"  Status: {status}")
+        logger.info(f"  Image: {image}")
+        logger.info(f"  Action: {action}")
+
+        # Optionally, log the full details for debugging purposes
+        logger.debug(f"Full event details: {data}")
+
+    except Exception as e:
+        logger.error(f"Error processing Docker event: {e}")
 
 @app.route("/api/assign_task", methods=["POST"])
 def manually_assign_task():
@@ -546,30 +657,6 @@ def create_task():
     except Exception as e:
         logger.error(f"Error creating task: {e}")
         return jsonify({"error": "Failed to create task"}), 500
-    
-    # # # Example request structure
-    # payload_text = {
-    #     "worker_id": "234",
-    #     "task_type": "container",
-    #     "job_details": {
-    #         "tenancyID": "tenant123a",
-    #         "containers": [
-    #             {
-    #                 "container_id": "af47632e-43b7-4474-82fa-475a24fe88d5",
-    #                 "docker_image": "nginx",
-    #                 "cpu_shares": 1,
-    #                 "mem_limit": 128,
-    #                 "container_name": "web-server1",
-    #                 "command": ["nginx", "-g", "daemon off;"],
-    #                 "working_dir": "/usr/share/nginx/html"
-    #             }
-    #         ]
-    #     }
-    # }
-
-
-
-
 
 
 if __name__ == "__main__":

worker.py

@@ -1,429 +0,0 @@
-import os
-from colorlog import ColoredFormatter
-import socketio
-import argparse
-import logging
-import json
-import asyncio
-import docker
-
-from worker_tasks.report import ReportTask
-from worker_tasks.ping import PingTask
-from worker_tasks.file_presence import FilePresenceTask
-from worker_tasks.container import ContainerTask
-
-from dotenv import load_dotenv
-
-# Configure logging
-DEBUG_SOCKETIO=False
-
-# Custom filter to include the function name
-class FunctionNameFilter(logging.Filter):
-    def filter(self, record):
-        record.funcName = record.funcName if hasattr(record, 'funcName') else '<unknown>'
-        return True
-
-# Define the colorized log format
-log_format = (
-    "%(log_color)s%(asctime)s - %(levelname)s - %(funcName)s - %(message)s"
-)
-date_format = "%Y-%m-%d %H:%M:%S"
-
-# TODO - Add fucntion name to logging output
-# Configure the formatter with colors
-formatter = ColoredFormatter(
-    log_format,
-    datefmt=date_format,
-    log_colors={
-        "DEBUG": "cyan",
-        "INFO": "green",
-        "WARNING": "yellow",
-        "ERROR": "red",
-        "CRITICAL": "bold_red",
-    },
-)
-
-# Configure the handler
-handler = logging.StreamHandler()
-handler.setFormatter(formatter)
-
-# Configure the logger
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-logger.addFilter(FunctionNameFilter())
-logger.addHandler(handler)
-
-# # Ensure the logs directory exists
-# log_dir = "logs"
-# if not os.path.exists(log_dir):
-#     os.makedirs(log_dir)
-
-# # Configure the file handler
-# file_handler = logging.FileHandler("logs/app.log")
-# file_handler.setFormatter(formatter)
-# file_handler.setLevel(logging.WARNING)
-# logger.addHandler(file_handler)
-
-
-class WorkerClient:
-    def __init__(self, worker_ID,worker_secret,server_URL):
-        self.worker_id = worker_ID
-        self.worker_secret = worker_secret
-        self.server_url = server_URL
-        if DEBUG_SOCKETIO:
-            self.sio = socketio.Client(logger=logger, engineio_logger=logger)
-        else:
-            self.sio = socketio.Client(logger=False, engineio_logger=False)
-        self.joined_server = False
-
-        # Initialize Docker client
-        self.docker_client = docker.from_env()
-
-        # Bind events
-        self.sio.on("connect", self.on_connect)
-        self.sio.on("connect_error", self.on_connect_error)
-        self.sio.on("disconnect", self.on_disconnect)
-        self.sio.on("task", self.handle_task)
-        self.sio.on("join_accept", self.on_join_accept)
-        self.sio.on("join_reject", self.on_join_reject)
-        self.sio.on("message", self.on_message)
-        self.sio.on("*", self.debug_all_events)
-
-    def on_connect(self):
-        logger.info("Connected to the server, requesting to join.")
-        self.send_join_request()
-        
-
-    def on_connect_error(self, data):
-        logger.error(f"Connection failed: {data}")
-        self.joined_server = False
-
-    def on_disconnect(self):
-        logger.info("Disconnected from the server.")
-        self.joined_server = False
-
-    def on_message(self,data):
-        logger.info(f"Message recieved {data}")
-
-    def on_join_accept(self,data):
-        logger.info("Join accepted.")
-        self.joined_server = True
-        # Start Docker event watcher
-        asyncio.create_task(self.watch_docker_events())
-
-    def on_join_reject(self,data):
-        logger.error("Join rejected.")
-        self.joined_server = False
-        exit()
-
-    def send_join_request(self):
-        """Notify the server about this worker (Initial join)."""
-        logger.info("Sending join request")
-        if not self.joined_server:
-            self.sio.emit("join_request", {"worker_id": self.worker_id,"worker_secret": self.worker_secret})
-            logger.info(f"Worker {self.worker_id} asked to join the server.")
-
-    def handle_task(self, data):
-        """Handle incoming tasks from the server."""
-        if not self.joined_server:
-            logger.error("Not joined to server. ignoring task")
-            return
-        task_type = data["type"]
-        task_id = data["task_id"]
-        job_details = json.loads(data["job_details"])
-        task_worker_id = data["worker_id"]
-
-        logger.info(f"Received task {task_id} of type '{task_type}' with job_details: {job_details}")
-        job_details_type=type(job_details)
-        logger.info(f"opbject type {job_details_type}")
-
-        try:
-            if task_type == "report":
-                result = ReportTask("",logger).Execute()
-            elif task_type == "ping":
-                result = PingTask(job_details,logger).Execute()
-            elif task_type == "file_presence":
-                result = FilePresenceTask(job_details,logger).Execute()
-            elif task_type == "container-create":
-                result = ContainerTask(job_details,logger).Create()
-            elif task_type == "container-delete":
-                result = ContainerTask(job_details,logger).Delete()
-            else:
-                raise ValueError(f"Unknown task type: {task_type}")
-
-            self.send_task_result(task_id, result, task_worker_id)
-        except Exception as e:
-            logger.error(f"Error processing task {task_id}: {e}")
-            self.send_task_result(  task_id, 
-                                    {"success": False, "response":str(e)},
-                                    task_worker_id)
-
-    def send_task_result(self, task_id, result, worker_id):
-        """Send task result back to the server."""
-        self.sio.emit("ack", {"task_id": task_id, "worker_id": worker_id, "result": result})
-        logger.info(f"Sent result for task {task_id}: {result}")
-
-    def debug_all_events(self, event, data):
-        """Debug all incoming data."""
-        logger.debug(f"Event: {event} | Data: {json.dumps(data, indent=2)}")
-
-    def start(self):
-        """Worker connects to the API server and processes tasks."""
-        try:
-            logger.info(f"Worker {self.worker_id} connecting to {self.server_url}...")
-            self.sio.connect(self.server_url)
-            self.sio.wait()
-        except Exception as e:
-            logger.error(f"Error: {e}")
-            self.sio.disconnect()
-
-    async def watch_docker_events(self):
-        """Watch Docker events and send them to the server for containers managed by this worker."""
-        logger.info("Starting Docker event watcher...")
-        try:
-            for event in self.docker_client.events(decode=True):
-                logger.debug(f"Caught docker event {event}")
-                if event['Type'] == 'container' and event['Action'] in ['start', 'stop']:
-                    container_id = event['id']
-                    try:
-                        # Fetch the container details to check its labels
-                        container = self.docker_client.containers.get(container_id)
-                        labels = container.attrs['Config']['Labels']
-                        
-                        # Check if the container is managed by this worker
-                        if labels and labels.get("managed_by") == "worker_agent":
-                            logger.info(f"Detected Docker event for managed container: {event}")
-                            await self.send_event_to_server(event)
-                    except docker.errors.NotFound:
-                        logger.warning(f"Container {container_id} not found. Skipping event.")
-                    except Exception as e:
-                        logger.error(f"Error fetching container details: {e}")
-        except Exception as e:
-            logger.error(f"Error watching Docker events: {e}")
-
-    async def send_event_to_server(self, event):
-        """Send Docker event to the server over WebSocket."""
-        try:
-            if self.joined_server:
-                self.sio.emit("docker_event", {"worker_id": self.worker_id, "event": event})
-                logger.info(f"Sent Docker event to server: {event}")
-        except Exception as e:
-            logger.error(f"Failed to send Docker event to server: {e}")
-
-
-def validate_json_input(text_input):
-    try:
-        # Step 1: Validate that it's valid JSON
-        data = json.loads(text_input)
-    except json.JSONDecodeError as e:
-        raise ValueError(f"Invalid JSON format: {e}")
-
-    # Step 2: Ensure 'tenancyID' is present
-    if 'tenancyID' not in data:
-        raise ValueError("Key 'tenancyID' is missing.")
-
-    # Step 3: Ensure 1 or more containers are present
-    if 'containers' not in data or not isinstance(data['containers'], list) or len(data['containers']) == 0:
-        raise ValueError("Key 'containers' is missing, not a list, or empty.")
-
-    # Step 4: For each container, ensure 'container_name' and 'docker_image' are present
-    for container in data['containers']:
-        if not isinstance(container, dict):
-            raise ValueError("Container is not a dictionary.")
-        if 'container_name' not in container or 'docker_image' not in container:
-            raise ValueError("Container is missing 'container_name' or 'docker_image'.")
-
-    # If all validations pass, return the parsed JSON object
-    return data
-
-
-
-def container_test():
-    logger.info("Running container test")
-    payload_text = '''{
-        "tenancyID": "tenant123a",
-        "containers": [
-            {
-                "container_id": "af47632e-43b7-4474-82fa-475a24fe88d5",
-                "docker_image": "nginx",
-                "cpu_shares": 1,
-                "mem_limit": 128,
-                "container_name": "web-server1",
-                "command": ["nginx", "-g", "daemon off;"],
-                "working_dir": "/usr/share/nginx/html"
-            },{
-                "container_id": "82c1c6ae-9dd4-41c0-ba05-45a086d58cf5",
-                "docker_image": "python:3.9-alpine",
-                "container_name": "python-webserver",
-                "cpu_shares": 1,
-                "mem_limit": 128,
-                "command": ["python", "-m", "http.server","82"],
-                "environment": {
-                    "DELAY_START_MSEC": "2000"
-                },
-                "restart_policy": {
-                    "Name": "always"
-                }
-            }
-        ]
-        
-    }'''
-    
-    # Parse payload
-    payload_json = validate_json_input(payload_text)
-    logger.debug(f"Container test payload {payload_json}")
-    # Initialize and execute ContainerTask
-    container_task = ContainerTask(payload_json, logger)
-    result = container_task.Execute()
-    logger.info(result)
-
-
-import os
-import json
-import requests
-from datetime import datetime
-
-def enroll_worker(region_id, enrollment_key):
-    # Configuration
-    API_URL = "http://127.0.0.1:5000/api/workload_hosts/enroll"
-
-    # Gather system information
-    def get_system_info():
-        try:
-            with open('/sys/class/dmi/id/sys_vendor', 'r') as f:
-                system_manufacturer = f.read().strip()
-        except:
-            system_manufacturer = "Unknown"
-
-        try:
-            with open('/sys/class/dmi/id/product_name', 'r') as f:
-                system_model = f.read().strip()
-        except:
-            system_model = "Unknown"
-
-        try:
-            with open('/sys/class/dmi/id/product_serial', 'r') as f:
-                physical_identifier = f.read().strip()
-        except:
-            physical_identifier = "Unknown"
-
-        try:
-            dcim_identifier = os.popen('dmidecode -s system-uuid').read().strip()
-        except:
-            dcim_identifier = "Unknown"
-
-        installed_date = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
-        hostname = os.uname().nodename
-
-        return {
-            "system_manufacturer": system_manufacturer,
-            "system_model": system_model,
-            "physical_identifier": physical_identifier,
-            "dcim_identifier": dcim_identifier,
-            "installed_date": installed_date,
-            "hostname": hostname
-        }
-
-    # Construct JSON payload
-    system_info = get_system_info()
-    payload = {
-        "region_id": region_id,
-        "region_enrollment_key": enrollment_key,
-        "system_manufacturer": system_info["system_manufacturer"],
-        "system_model": system_info["system_model"],
-        "physical_identifier": system_info["physical_identifier"],
-        "dcim_identifier": system_info["dcim_identifier"],
-        "installed_date": system_info["installed_date"],
-        "hostname": system_info["hostname"]
-    }
-
-    # Send enrollment request
-    response = requests.post(API_URL, json=payload, headers={"Content-Type": "application/json"})
-
-    # Handle response
-    if response.status_code != 201:
-        print(f"Enrollment failed: {response.status_code} - {response.text}")
-        exit(1)
-
-    # Parse response JSON
-    try:
-        response_data = response.json()
-        worker_id = response_data.get("worker_id")
-        worker_secret = response_data.get("worker_secret")
-
-        if not worker_id or not worker_secret:
-            print("Enrollment failed: Worker_ID or Worker_Secret not found in response")
-            exit(1)
-
-        # Ensure .env file exists before appending
-        if not os.path.exists(".env"):
-            open(".env", "w").close()  # Create an empty .env file if it doesn't exist
-
-        # Append worker credentials to .env
-        with open(".env", "a") as env_file:
-            env_file.write(f"WORKER_ID={worker_id}\n")
-            env_file.write(f"WORKER_SECRET={worker_secret}\n")
-
-
-        logger.info("Enrollment successful. Worker_ID and Worker_Secret written to .env file.")
-        return response_data
-    
-    except json.JSONDecodeError:
-        print("Enrollment failed: Invalid JSON response from server")
-        exit(1)
-
-if __name__ == "__main__":
-    load_dotenv()
-
-    parser = argparse.ArgumentParser(description="Worker CLI")
-    parser.add_argument("--server-url", help="WebSocket Server URL (e.g., http://localhost:5000)", default=os.getenv("SERVER_URL"))
-    parser.add_argument("--region-id", help="Region ID(Only required during enrollment)", default=os.getenv("REGION_ID"))
-    parser.add_argument("--region-enrollment-key", help="Region Enrollment Key", default=os.getenv("REGION_ENROLLMENT_KEY"))
-    parser.add_argument("--container-test", action="store_true", help="Run container test")
-    args = parser.parse_args()
-
-    # Verbose logging of environment variables
-    logger.info("Starting Worker CLI")
-    logger.info(f"SERVER_URL: {args.server_url if args.server_url else 'Not Provided'}")
-    logger.info(f"REGION_ID: {args.region_id if args.region_id else 'Not Provided'}")
-    logger.info(f"REGION_ENROLLMENT_KEY: {args.region_enrollment_key if args.region_enrollment_key else 'Not Provided'}")
-
-    if args.container_test:
-        logger.info("Running container test...")
-        container_test()
-        exit()
-
-    # Ensure required arguments are provided
-    if not args.server_url:
-        parser.error("--server-url are required when --container-test is not used and not defined in .env.")
-
-
-
-    # Enroll worker only if WORKER_ID and WORKER_SECRET are not in .env
-    worker_secret = os.getenv("WORKER_SECRET")
-    worker_id = os.getenv("WORKER_ID")
-    if not worker_id or not worker_secret:
-        logger.info("Worker ID or Worker Secret missing. Enrolling worker...")
-        #Region ID is only requried for enrollment
-        if not args.region_id:
-            parser.error("--region-id must be provided either in the .env file or via CLI.")
-        if not args.region_enrollment_key:
-            parser.error("--region-enrollment-key must be provided either in the .env file or via CLI.")
-
-        enroll_worker(args.region_id,args.region_enrollment_key)
-        load_dotenv()
-    else:
-        logger.info("Worker ID and Worker Secret found. Skipping enrollment.")
-
-
-    worker_ID=os.getenv("WORKER_ID")
-    worker_secret=os.getenv("WORKER_SECRET")
-    server_URL=os.getenv("SERVER_URL") if os.getenv("SERVER_URL") else args.server_url
-
-    # Create and start the worker
-    logger.info("Starting Worker with the folowing params...")
-    logger.info(f"WORKER_ID: {worker_ID}")
-    logger.info(f"WORKER_SECRET: {worker_secret}")
-    logger.info(f"SERVER_URL: {server_URL}")
-    worker = WorkerClient(worker_ID,worker_secret,server_URL)
-    worker.start()

worker/docker.py

@@ -0,0 +1,59 @@
+import docker
+from logger import logger
+
+class DockerMonitor:
+    def __init__(self, event_queue):
+        self.event_queue = event_queue
+        self.docker_client = docker.from_env()
+        self.running = True
+    
+    def run(self):
+        """Monitor Docker events and extract the 'managed_by' tag if it exists."""
+        logger.info("Docker monitor started")
+        
+        # Get events from Docker
+        events = self.docker_client.events(decode=True)
+        
+        for event in events:
+            if not self.running:
+                break
+                
+            if event.get('Type') == 'container':
+                # Process container events (start, stop, die, etc.)
+                status = event.get('status')
+                container_id = event.get('id')
+                container_name = ""
+                
+                if status in ['start', 'stop', 'die', 'create', 'destroy']:
+                    # Extract the 'managed_by' tag from the container labels (if it exists)
+                    managed_by = None
+                    container_attributes = event.get('Actor', {}).get('Attributes', {})
+                    if container_attributes:
+                        managed_by = container_attributes.get('managed_by')
+                        container_name = container_attributes.get('name')
+                        system_container_id = container_attributes.get('system_container_id')
+                    
+                    if not managed_by=="worker_agent":
+                        logger.debug(f"Ignoring {container_id}, not managed by us")
+                        continue
+                    
+                    # Prepare the event data
+                    event_data = {
+                        'source': 'docker',
+                        'event_type': status,
+                        'container_id': container_id,
+                        'system_container_id': system_container_id,
+                        'container_name': container_name,
+                        'timestamp': event.get('time'),
+                        'details': event,
+                        'managed_by': managed_by  # Add the 'managed_by' tag to the payload
+                    }
+                    
+                    # Put event in the queue for the worker client
+                    self.event_queue.put(event_data)
+                    logger.info(f"Added Docker event to queue: {status} for container {container_id}. Managed by: {managed_by}")
+    
+    def stop(self):
+        """Stop the Docker monitor"""
+        self.running = False
+        self.docker_client.close()

worker/libvirt.py

@@ -0,0 +1,215 @@
+from logger import logger
+import libvirt
+import time
+import xml.etree.ElementTree as ET
+import threading
+
+class LibvirtMonitor:
+    def __init__(self, event_queue):
+        self.event_queue = event_queue
+        self.conn = None
+        self.running = True
+        self.event_impl = None
+    
+    def connect_libvirt(self):
+        """Connect to libvirt"""
+        try:
+            # Try to connect to local libvirt
+            self.conn = libvirt.open('qemu:///system')
+            
+            # Initialize the event implementation
+            libvirt.virEventRegisterDefaultImpl()
+            self.event_impl = True
+            
+            return True
+        except libvirt.libvirtError as e:
+            logger.error(f"Failed to connect to libvirt: {e}")
+            return False
+
+    def callback_lifecycle(self, conn, dom, event, detail, opaque):
+        """Callback for domain lifecycle events"""
+        event_names = {
+            libvirt.VIR_DOMAIN_EVENT_STARTED: 'started',
+            libvirt.VIR_DOMAIN_EVENT_STOPPED: 'stopped',
+            libvirt.VIR_DOMAIN_EVENT_SUSPENDED: 'suspended',
+            libvirt.VIR_DOMAIN_EVENT_RESUMED: 'resumed',
+            libvirt.VIR_DOMAIN_EVENT_SHUTDOWN: 'shutdown',
+            libvirt.VIR_DOMAIN_EVENT_DEFINED: 'defined',
+            libvirt.VIR_DOMAIN_EVENT_UNDEFINED: 'undefined'
+        }
+        
+        event_name = event_names.get(event, f'unknown-{event}')
+        
+        # Get domain information
+        try:
+            domain_name = dom.name()
+            domain_id = dom.ID()
+            
+            # Check metadata for 'vm_id'
+            vm_id = self.get_vm_id_from_metadata(dom)
+            if vm_id is None:
+                logger.warning("NO VMID found, likely not a VM managed by us, ignoring")
+                return 0
+            logger.debug(f"Found VM_ID {vm_id}")
+
+            event_data = {
+                'source': 'libvirt',
+                'event_type': event_name,
+                'system_vm_id': vm_id,
+                'libvirt_vm_id': domain_id,
+                'timestamp': time.time()
+            }
+            
+        
+            # Get the VNC port if available
+            vnc_port = self.get_vnc_port(dom)
+            if vnc_port is not None:
+                event_data['vnc_port'] = vnc_port
+            
+
+
+
+            # Put event in the queue for the worker client
+            self.event_queue.put(event_data)
+            logger.info(f"Added libvirt event to queue: {event_name} for domain {domain_name}")
+            
+        except libvirt.libvirtError as e:
+            logger.error(f"Error getting domain info: {e}")
+    
+    def get_vnc_port(self, dom):
+        """Extract the VNC port from the domain XML"""
+        try:
+            xml_desc = dom.XMLDesc(0)
+            root = ET.fromstring(xml_desc)
+            
+            # Find the graphics element with type 'vnc'
+            graphics = root.find(".//graphics[@type='vnc']")
+            if graphics is not None:
+                port = graphics.get('port')
+                if port is not None:
+                    return int(port)
+        except Exception as e:
+            logger.error(f"Error extracting VNC port: {e}")
+        return None
+    
+    def get_vm_id_from_metadata(self, dom):
+        """
+        Extract the VM ID from the XML configuration.
+        
+        Args:
+            xml_config (str): XML configuration of the VM.
+            logger: Logger instance for logging.
+        
+        Returns:
+            str: The VM ID if found, otherwise None.
+        """
+        try:
+            # Parse the XML configuration
+            xml_desc = dom.XMLDesc(0)
+            root = ET.fromstring(xml_desc)
+            
+            # Define the custom namespace
+            namespaces = {
+                "custom": "http://example.com/xmlns/libvirt/custom"
+            }
+            
+            # Find the metadata section
+            metadata = root.find(".//metadata")
+            
+            if metadata is None:
+                logger.warning("Metadata section not found in the XML configuration.")
+                return None
+            
+            logger.debug(f"Metadata section: {ET.tostring(metadata)}")
+            
+            # Look for the 'vm_id' tag within the custom namespace
+            virtual_machine_id_tag = metadata.find(".//custom:virtual_machine_id", namespaces)
+            logger.debug(f"VM ID tag: {virtual_machine_id_tag}")
+            
+            if virtual_machine_id_tag is not None:
+                return virtual_machine_id_tag.text
+            
+            logger.warning("VM ID not found in the metadata section.")
+            return None
+        
+        except Exception as e:
+            logger.error(f"Error extracting VM ID from XML: {e}")
+            raise
+                
+
+
+    def run(self):
+        """Monitor libvirt events"""
+        logger.info("Libvirt monitor started")
+        
+        try:
+            if not self.connect_libvirt():
+                logger.error("Failed to start libvirt monitor")
+                return
+            
+            # Register for domain lifecycle events
+            cb_id = self.conn.domainEventRegisterAny(
+                None,
+                libvirt.VIR_DOMAIN_EVENT_ID_LIFECYCLE,
+                self.callback_lifecycle,
+                None
+            )
+            
+            
+            def event_loop_thread():
+                while self.running:
+                    try:
+                        libvirt.virEventRunDefaultImpl()
+                        time.sleep(0.1)  # Small delay to prevent CPU thrashing
+                    except Exception as e:
+                        logger.error(f"Error in libvirt event loop: {e}")
+                        time.sleep(1)  # Longer delay after error
+            
+            # Start event loop thread
+            event_thread = threading.Thread(target=event_loop_thread)
+            event_thread.daemon = True
+            event_thread.start()
+            
+            # Main monitor loop - periodically check domains
+            while self.running:
+                try:
+                    # Poll for active domains as a fallback mechanism
+                    domains = self.conn.listAllDomains()
+                    for domain in domains:
+                        state, reason = domain.state()
+                        domain_name = domain.name()
+                        # Just log the current state, don't generate events
+                        # This is just to keep the connection active
+                        # logger.debug(f"Domain {domain_name} state: {state}")
+                except libvirt.libvirtError as e:
+                    logger.error(f"Error polling libvirt domains: {e}")
+                    # Try to reconnect
+                    try:
+                        if self.conn:
+                            self.conn.close()
+                        self.connect_libvirt()
+                    except Exception as e:
+                        logger.error(f"Failed to reconnect to libvirt: {e}")
+                        
+                time.sleep(5)  # Check every 5 seconds
+                
+            # Cleanup
+            try:
+                if cb_id >= 0:
+                    self.conn.domainEventDeregisterAny(cb_id)
+            except Exception:
+                pass
+            
+        except Exception as e:
+            logger.error(f"Fatal error in libvirt monitor: {e}")
+        finally:
+            # Close connection
+            if self.conn:
+                try:
+                    self.conn.close()
+                except Exception:
+                    pass
+    
+    def stop(self):
+        """Stop the libvirt monitor"""
+        self.running = False

worker/worker.py

@@ -0,0 +1,193 @@
+import socketio
+from logger import logger
+import json
+import asyncio
+from worker_tasks.container import ContainerTask
+from worker_tasks.file_presence import FilePresenceTask
+from worker_tasks.libvirt import LibvirtVirtualMachineTask
+from worker_tasks.ping import PingTask
+from worker_tasks.report import ReportTask
+
+class WorkerClient:
+    def __init__(self, worker_ID, worker_secret, server_URL, event_queue=None,DEBUG_SOCKETIO=False):
+        self.worker_id = worker_ID
+        self.worker_secret = worker_secret
+        self.server_url = server_URL
+        self.event_queue = event_queue
+        self.running = True
+        self.libvirt_config={
+            "default_volume_path": "/tmp"
+        }
+        
+        if DEBUG_SOCKETIO:
+            self.sio = socketio.AsyncClient(logger=logger, engineio_logger=logger)
+        else:
+            self.sio = socketio.AsyncClient(logger=False, engineio_logger=False)
+        self.joined_server = False
+
+        # Bind events
+        self.sio.on("connect", self.on_connect)
+        self.sio.on("connect_error", self.on_connect_error)
+        self.sio.on("disconnect", self.on_disconnect)
+        self.sio.on("task", self.handle_task)
+        self.sio.on("join_accept", self.on_join_accept)
+        self.sio.on("join_reject", self.on_join_reject)
+        self.sio.on("message", self.on_message)
+        self.sio.on("*", self.debug_all_events)
+
+    async def on_connect(self):
+        logger.info("Connected to the server, requesting to join.")
+        await self.send_join_request()
+
+    async def on_connect_error(self, data):
+        logger.error(f"Connection failed: {data}")
+        self.joined_server = False
+
+    async def on_disconnect(self):
+        logger.info("Disconnected from the server.")
+        self.joined_server = False
+
+    async def on_message(self, data):
+        logger.info(f"Message received: {data}")
+
+    async def on_join_accept(self, data):
+        logger.info("Join accepted.")
+        self.joined_server = True
+
+    async def on_join_reject(self, data):
+        logger.error("Join rejected.")
+        self.joined_server = False
+        exit()
+
+    async def send_join_request(self):
+        """Notify the server about this worker (Initial join)."""
+        logger.info("Sending join request")
+        if not self.joined_server:
+            await self.sio.emit("join_request", {"worker_id": self.worker_id, "worker_secret": self.worker_secret})
+            logger.info(f"Worker {self.worker_id} asked to join the server.")
+
+    async def handle_task(self, data):
+        """Handle incoming tasks from the server."""
+        if not self.joined_server:
+            logger.error("Not joined to server. ignoring task")
+            return
+        task_type = data["type"]
+        task_id = data["task_id"]
+        job_details = json.loads(data["job_details"])
+        task_worker_id = data["worker_id"]
+
+        logger.info(f"Received task {task_id} of type '{task_type}' with job_details: {job_details}")
+
+        try:
+            if task_type == "report":
+                result = ReportTask("", logger).Execute()
+            elif task_type == "ping":
+                result = PingTask(job_details, logger).Execute()
+            elif task_type == "file_presence":
+                result = FilePresenceTask(job_details, logger).Execute()
+            elif task_type == "container-create":
+                result = ContainerTask(job_details, logger).Create()
+            elif task_type == "container-delete":
+                result = ContainerTask(job_details, logger).Delete()
+            elif task_type == "virtual-machine-create":
+                result = LibvirtVirtualMachineTask(job_details,self.libvirt_config, logger).execute()
+            elif task_type == "virtual-machine-delete":
+                result = LibvirtVirtualMachineTask(job_details,self.libvirt_config, logger).execute()
+            else:
+                raise ValueError(f"Unknown task type: {task_type}")
+
+
+            await self.send_task_result(task_id, result, task_worker_id)
+        except Exception as e:
+            logger.error(f"Error processing task {task_id}: {e}")
+            await self.send_task_result(task_id, {"success": False, "response": str(e)}, task_worker_id)
+
+    async def send_task_result(self, task_id, result, worker_id):
+        """Send task result back to the server."""
+        await self.sio.emit("ack", {"task_id": task_id, "worker_id": worker_id, "result": result})
+        logger.info(f"Sent result for task {task_id}: {result}")
+
+    async def send_event(self, event_data):
+        """Send monitoring event to the server, handling libvirt and docker events separately."""
+        if not self.joined_server:
+            return
+
+        source = event_data.get('source', 'unknown')
+        event_type = event_data.get('event_type', 'event')
+
+        # Construct the base event payload
+        payload = {
+            "worker_id": self.worker_id,
+            "type": f"{source}_{event_type}",
+            "details": event_data
+        }
+
+        # Handle libvirt events
+        if source == "libvirt":
+            payload["details"] = {
+                "system_vm_id": event_data.get("system_vm_id"),
+                "libvirt_vm_id": event_data.get("libvirt_vm_id"),
+                "timestamp": event_data.get("timestamp"),
+            }
+
+            # Emit the event to the server
+            await self.sio.emit("libvirt_event", payload)
+            logger.info(f"Sent libvirt event: {event_type}\n{payload}")
+
+
+        # Handle docker events
+        elif source == "docker":
+            payload["details"] = {
+                "container_id": event_data.get("container_id"),
+                "system_container_id": event_data.get("system_container_id"),
+                "container_name": event_data.get("container_name"),
+                "timestamp": event_data.get("timestamp"),
+                "status": event_data.get("details", {}).get("status"),
+                "image": event_data.get("details", {}).get("from"),
+                "action": event_data.get("details", {}).get("Action")
+            }
+
+            # Emit the event to the server
+            await self.sio.emit("docker_event", payload)
+            logger.info(f"Sent {source} docker event: {event_type}\n{payload}")
+
+    async def debug_all_events(self, event, data):
+        """Debug all incoming data."""
+        logger.debug(f"Event: {event} | Data: {json.dumps(data, indent=2)}")
+
+    async def process_event_queue(self):
+        """Process events from the queue and send them to the server."""
+        if not self.event_queue:
+            return
+            
+        while self.running:
+            try:
+                # Non-blocking check for events
+                if not self.event_queue.empty():
+                    event = self.event_queue.get(block=False)
+                    logger.info(f"Processing event from queue: {event}")
+                    await self.send_event(event)
+                await asyncio.sleep(0.1)  # Small delay to prevent CPU thrashing
+            except Exception as e:
+                logger.error(f"Error in event processing: {e}")
+    
+    async def start(self):
+        """Worker connects to the API server and processes tasks."""
+        try:
+            logger.info(f"Worker {self.worker_id} connecting to {self.server_url}...")
+            await self.sio.connect(self.server_url)
+            
+            # Start the event queue processor if we have an event queue
+            if self.event_queue:
+                event_processor = asyncio.create_task(self.process_event_queue())
+                await self.sio.wait()
+                event_processor.cancel()
+            else:
+                await self.sio.wait()
+        except Exception as e:
+            logger.error(f"Error: {e}")
+            await self.sio.disconnect()
+    
+    def stop(self):
+        """Stop the worker client."""
+        self.running = False

worker2.py

@@ -0,0 +1,280 @@
+import multiprocessing as mp
+from multiprocessing import Queue
+import json
+import signal
+import sys
+import asyncio
+import docker
+import os
+import argparse
+import requests
+from datetime import datetime
+from dotenv import load_dotenv
+from logger import logger
+
+from worker.docker import DockerMonitor
+from worker.libvirt import LibvirtMonitor
+from worker.worker import WorkerClient
+
+# Debug flag
+DEBUG_SOCKETIO = False
+
+
+def enroll_worker(region_id, api_base_url="http://127.0.0.1:5000"):
+    """Enroll the worker with the server and get worker credentials"""
+    API_URL = f"{api_base_url}/api/workload_hosts/enroll"
+
+    # Gather system information
+    def get_system_info():
+        try:
+            with open('/sys/class/dmi/id/sys_vendor', 'r') as f:
+                system_manufacturer = f.read().strip()
+        except:
+            system_manufacturer = "Unknown"
+
+        try:
+            with open('/sys/class/dmi/id/product_name', 'r') as f:
+                system_model = f.read().strip()
+        except:
+            system_model = "Unknown"
+
+        try:
+            with open('/sys/class/dmi/id/product_serial', 'r') as f:
+                physical_identifier = f.read().strip()
+        except:
+            physical_identifier = "Unknown"
+
+        try:
+            dcim_identifier = os.popen('dmidecode -s system-uuid').read().strip()
+        except:
+            dcim_identifier = "Unknown"
+
+        installed_date = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+        hostname = os.uname().nodename
+
+        return {
+            "system_manufacturer": system_manufacturer,
+            "system_model": system_model,
+            "physical_identifier": physical_identifier,
+            "dcim_identifier": dcim_identifier,
+            "installed_date": installed_date,
+            "hostname": hostname
+        }
+
+    # Construct JSON payload
+    system_info = get_system_info()
+    payload = {
+        "region_id": region_id,
+        "system_manufacturer": system_info["system_manufacturer"],
+        "system_model": system_info["system_model"],
+        "physical_identifier": system_info["physical_identifier"],
+        "dcim_identifier": system_info["dcim_identifier"],
+        "installed_date": system_info["installed_date"],
+        "hostname": system_info["hostname"]
+    }
+
+    # Send enrollment request
+    logger.info(f"Sending enrollment request to {API_URL}")
+    response = requests.post(API_URL, json=payload, headers={"Content-Type": "application/json"})
+
+    # Handle response
+    if response.status_code != 201:
+        logger.error(f"Enrollment failed: {response.status_code} - {response.text}")
+        exit(1)
+
+    # Parse response JSON
+    try:
+        response_data = response.json()
+        worker_id = response_data.get("worker_id")
+        worker_secret = response_data.get("worker_secret")
+
+        if not worker_id or not worker_secret:
+            logger.error("Enrollment failed: Worker_ID or Worker_Secret not found in response")
+            exit(1)
+
+        # Ensure .env file exists before appending
+        if not os.path.exists(".env"):
+            open(".env", "w").close()  # Create an empty .env file if it doesn't exist
+
+        # Append worker credentials to .env
+        with open(".env", "a") as env_file:
+            env_file.write(f"WORKER_ID={worker_id}\n")
+            env_file.write(f"WORKER_SECRET={worker_secret}\n")
+
+        logger.info("Enrollment successful. Worker_ID and Worker_Secret written to .env file.")
+        return response_data
+    
+    except json.JSONDecodeError:
+        logger.error("Enrollment failed: Invalid JSON response from server")
+        exit(1)
+
+
+def container_test():
+    """Run a simple test to verify Docker functionality"""
+    try:
+        client = docker.from_env()
+        containers = client.containers.list()
+        logger.info(f"Container test successful. Found {len(containers)} running containers.")
+        for container in containers:
+            logger.info(f"Container: {container.name} (ID: {container.short_id})")
+        return True
+    except Exception as e:
+        logger.error(f"Container test failed: {e}")
+        return False
+
+
+def libvirt_test():
+    """Run a simple test to verify libvirt functionality"""
+    try:
+        conn = libvirt.open('qemu:///system')
+        domains = conn.listAllDomains()
+        logger.info(f"Libvirt test successful. Found {len(domains)} domains.")
+        for domain in domains:
+            logger.info(f"Domain: {domain.name()} (ID: {domain.ID()})")
+        conn.close()
+        return True
+    except Exception as e:
+        logger.error(f"Libvirt test failed: {e}")
+        return False
+
+
+# Worker process function
+def worker_process_func(event_queue, worker_id, worker_secret, server_url):
+    """Function to run the worker client in a separate process"""
+    worker = WorkerClient(worker_id, worker_secret, server_url, event_queue,DEBUG_SOCKETIO)
+    
+    # Run the asyncio event loop
+    async def main():
+        await worker.start()
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        pass
+    except Exception as e:
+        logger.error(f"Worker process error: {e}")
+    finally:
+        worker.stop()
+
+
+def main():
+    # Load environment variables
+    load_dotenv()
+    
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(description="Worker CLI with Docker and Libvirt monitoring")
+    parser.add_argument("--server-url", help="WebSocket Server URL (e.g., http://localhost:5000)", default=os.getenv("SERVER_URL"))
+    parser.add_argument("--region-id", help="Region ID (Only required during enrollment)", default=os.getenv("REGION_ID"))
+    parser.add_argument("--container-test", action="store_true", help="Run container test and exit")
+    parser.add_argument("--libvirt-test", action="store_true", help="Run libvirt test and exit")
+    parser.add_argument("--no-docker", action="store_true", help="Disable Docker monitoring")
+    parser.add_argument("--no-libvirt", action="store_true", help="Disable Libvirt monitoring")
+    parser.add_argument("--api-base-url", help="Base URL for API (used for enrollment)", default=os.getenv("API_BASE_URL", "http://127.0.0.1:5000"))
+    
+    args = parser.parse_args()
+    args = parser.parse_args()
+
+    # Verbose logging of environment variables
+    logger.info("Starting Worker CLI with monitoring capabilities")
+    logger.info(f"SERVER_URL: {args.server_url if args.server_url else 'Not Provided'}")
+    logger.info(f"REGION_ID: {args.region_id if args.region_id else 'Not Provided'}")
+    logger.info(f"API_BASE_URL: {args.api_base_url}")
+
+    # Run tests if requested
+    if args.container_test:
+        logger.info("Running container test...")
+        container_test()
+        exit()
+    
+    if args.libvirt_test:
+        logger.info("Running libvirt test...")
+        libvirt_test()
+        exit()
+
+    # Ensure required arguments are provided
+    if not args.server_url:
+        parser.error("--server-url is required when not defined in .env.")
+
+    # Enroll worker only if WORKER_ID and WORKER_SECRET are not in .env
+    worker_secret = os.getenv("WORKER_SECRET")
+    worker_id = os.getenv("WORKER_ID")
+    if not worker_id or not worker_secret:
+        logger.info("Worker ID or Worker Secret missing. Enrolling worker...")
+        # Region ID is only required for enrollment
+        if not args.region_id:
+            parser.error("--region-id must be provided either in the .env file or via CLI.")
+
+        enroll_worker(args.region_id, args.api_base_url)
+        # Reload environment variables to get the new worker credentials
+        load_dotenv()
+    else:
+        logger.info("Worker ID and Worker Secret found. Skipping enrollment.")
+
+    # Get worker credentials
+    worker_id = os.getenv("WORKER_ID")
+    worker_secret = os.getenv("WORKER_SECRET")
+    server_url = os.getenv("SERVER_URL") if os.getenv("SERVER_URL") else args.server_url
+
+    # Create a queue for inter-process communication
+    event_queue = Queue()
+    
+    # Create monitoring processes
+    processes = []
+    monitors = []
+    
+    # Create worker process
+    logger.info("Starting Worker with the following params...")
+    logger.info(f"WORKER_ID: {worker_id}")
+    logger.info(f"WORKER_SECRET: {worker_secret}")
+    logger.info(f"SERVER_URL: {server_url}")
+    worker_proc = mp.Process(target=worker_process_func, args=(event_queue, worker_id, worker_secret, server_url))
+    processes.append(worker_proc)
+    
+    # Create Docker monitor process if enabled
+    if not args.no_docker:
+        logger.info("Docker monitoring enabled")
+        docker_monitor = DockerMonitor(event_queue)
+        docker_proc = mp.Process(target=docker_monitor.run)
+        processes.append(docker_proc)
+        monitors.append(docker_monitor)
+    else:
+        logger.warning("Docker monitoring disabled")
+    
+    # Create Libvirt monitor process if enabled
+    if not args.no_libvirt:
+        logger.info("Libvirt monitoring enabled")
+        libvirt_monitor = LibvirtMonitor(event_queue)
+        libvirt_proc = mp.Process(target=libvirt_monitor.run)
+        processes.append(libvirt_proc)
+        monitors.append(libvirt_monitor)
+    else:
+        logger.warning("Libvirt monitoring disabled")
+    
+    # Start all processes
+    for process in processes:
+        process.start()
+    
+    # Handle graceful shutdown
+    def signal_handler(sig, frame):
+        logger.info("Shutting down...")
+        for monitor in monitors:
+            monitor.stop()
+        
+        for process in processes:
+            process.join(timeout=5)
+            if process.is_alive():
+                process.terminate()
+        
+        sys.exit(0)
+    
+    # Register signal handlers
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+    
+    # Wait for processes to finish
+    for process in processes:
+        process.join()
+
+
+if __name__ == "__main__":
+    main()

worker_tasks/container.py

@@ -1,6 +1,5 @@
 import docker
 import json
-import logging
 from logging import Logger
 
 
@@ -15,8 +14,7 @@ class ContainerTask:
         self.params = params
         self.logger = logger
 
-        container_count = len(params['containers'])
-        logger.info(f"ContainerTask initialized successfully, preparing to process {container_count} containers")
+        logger.info(f"ContainerTask initialized successfully")
 
 
 
@@ -84,21 +82,6 @@ class ContainerTask:
             return None  # Master container not found
 
 
-            
-
-        
-    def check_or_create_nscontroller(self, client, tenancyID):
-        """Check if a namespace controller exists for the given tenancy ID. If not, create one."""
-        nscontroller_name = f"nscontroller-{tenancyID}"
-        self.logger.info(f"Checking for namespace controller '{nscontroller_name}'...")
-        containers = client.containers.list(all=True, filters={"name": nscontroller_name})
-        if not containers:
-            self.logger.info(f"Namespace controller '{nscontroller_name}' does not exist. Creating...")
-            client.containers.run("busybox", name=nscontroller_name, detach=True, command="sleep infinite")
-            self.logger.info(f"Namespace controller '{nscontroller_name}' created successfully.")
-        else:
-            self.logger.info(f"Namespace controller '{nscontroller_name}' already exists.")
-        return nscontroller_name
 
     def delete_container(self, client, container_name):
         """Delete the specified container if it exists."""
@@ -113,12 +96,12 @@ class ContainerTask:
         else:
             self.logger.info(f"Container '{container_name}' does not exist. No action needed.")
 
-    def launch_container(self, client, container, nscontroller_name):
+    def launch_container(self, client, container):
         """Launch the container as specified in the container variable, attaching it to the namespace controller."""
         container_name = container['container_id']
         
         # Check if the container is marked for deletion
-        if container.get('deleted', False):
+        if container.get('desired_state', False):
             self.logger.info(f"Container '{container_name}' is marked for deletion. Ensuring it is not present...")
             self.delete_container(client, container_name)
             return  # Exit the function as no further action is needed for deleted containers
@@ -167,15 +150,37 @@ class ContainerTask:
                     
                     self.logger.info(f"Removing container '{container_name}' and recreating with the new configuration...")
                     self.delete_container(client, container_name)
+        #Check if the NSController container exists
         
-        self.logger.info(f"Launching container '{container_name}'...")
-        # Add the logic to launch the container here
+        
+        nscontroller_container_name=container['NSController_launchparams']['container_name']
+        self.logger.debug(f"Name of NSController is {nscontroller_container_name}")
+        running_NSController = client.containers.list(all=True, filters={"name": nscontroller_container_name})
+        self.logger.debug(f"{running_NSController}")
+        if not running_NSController:
+            self.logger.info("NSController does not exist, creating it")
+            nscontroller_container_config = {
+                "image": container['NSController_launchparams']['docker_image'],
+                "name": container['NSController_launchparams']['container_name'],
+                "command": container['NSController_launchparams']['command'],
+                "network": "none",
+                "detach": True,
+                "labels": {
+                    "managed_by": "worker_agent",
+                    "system_container_id": container['NSController_launchparams']['container_id']
+                } 
+            }
 
+            # Launch the container
+            client.containers.run(**nscontroller_container_config)
+            self.logger.info(f"Container '{nscontroller_container_config}' launched successfully.")
+
+        self.logger.info(f"Launching container '{container_name}'...")
         # Prepare container configuration
         container_config = {
             "image": container['docker_image'],
             "name": container_name,
-            "network": f"container:{nscontroller_name}",
+            "network": f"container:{nscontroller_container_name}",
             "detach": True,
             "labels": {"managed_by": "worker_agent"}  # Add a label to identify containers started by this worker
         }
@@ -211,16 +216,15 @@ class ContainerTask:
             client = docker.from_env()
             self.logger.info("Docker client initialized.")
 
-            # Check or create namespace controller
-            nscontroller_name = self.check_or_create_nscontroller(client, self.params['tenancyID'])
-
             # Launch the container
-            all_containers= json.loads(self.params['containers'])
+            _container= json.loads(self.params['container'])
 
-            for _container in all_containers:
-                self.logger.info(_container)
-                create_result=self.launch_container(client, _container, nscontroller_name)
-                #TODO - Track the resultsof the tasks so we can rollback\delete if any subsequent part of the task fails
+            self.logger.info(_container)
+            # Is this container a NS controller or regular container?
+            # container_workload_type=self.params['container']
+        
+            create_result=self.launch_container(client, _container)
+            #TODO - Track the resultsof the tasks so we can rollback\delete if any subsequent part of the task fails
 
             self.logger.info("ContainerTask executed successfully.")
 
@@ -235,7 +239,7 @@ class ContainerTask:
 
     def Delete(self):
         """Delete the specified container and its namespace controller if it's the last one."""
-        container_name=self.params['containers'][0]['container_id']
+        container_name=self.params['container'][0]['container_id']
         self.logger.info(f"Executing ContainerTask - Delete for container '{container_name}'...")
         try:
             # Initialize Docker client

worker_tasks/libvirt.py

@@ -1,31 +1,46 @@
+import re
 import libvirt
 import difflib
+import json
+import os
+import subprocess
 from logging import Logger
+import xml.etree.ElementTree as ET
+from xml.dom import minidom
+
+import requests
+import hashlib
+import gzip
+import shutil
 
 
-class LibvirtVMTask:
-    def __init__(self, params, logger):
+class LibvirtVirtualMachineTask:
+    def __init__(self, params, config, logger):
         """
-        Initialize the LibvirtVMTask.
+        Initialize the LibvirtVirtualMachineTask.
 
         Args:
-            params (dict): Contains 'vm_name', 'desired_state', and optional 'action' and 'xml_config'.
+            params (dict): Contains 'virtual_machine_id', 'desired_state', optional 'action' and 'virtual_machine_config'/'xml_config'.
+                virtual_machine_config (dict): VirtualMachine configuration details including CPU, RAM, networks, volumes.
+                xml_config (str): Optional direct XML configuration (used if virtual_machine_config not provided).
+            config (dict): Configuration including default volume path.
             logger (Logger): Logger instance for logging.
         """
         if not isinstance(logger, Logger):
             raise TypeError("Logger must be an instance of the logging.Logger class.")
         self.logger = logger
 
-        self.logger.info("Initializing LibvirtVMTask")
+        self.logger.info("Initializing LibvirtVirtualMachineTask")
 
         # Validate params
         required_params = {
-            "vm_name": str,
+            "virtual_machine_id": str,
             "desired_state": str,  # e.g., 'running', 'stopped', 'deleted'
         }
         optional_params = {
             "action": str,  # 'reboot', 'start', 'stop'
-            "xml_config": str,  # Only required for 'create'
+            "xml_config": str,  # Direct XML configuration
+            "virtual_machine_config": dict,  # JSON configuration for VirtualMachine details
         }
 
         if not isinstance(params, dict):
@@ -45,15 +60,350 @@ class LibvirtVMTask:
                 self.logger.error(f"Parameter '{key}' must be of type {expected_type.__name__}.")
                 raise TypeError(f"Parameter '{key}' must be of type {expected_type.__name__}.")
 
-        self.vm_name = params["vm_name"]
+        self.virtual_machine_id = params["virtual_machine_id"]
         self.desired_state = params["desired_state"]
         self.action = params.get("action")
+
+        # Handle VirtualMachine configuration
+        self.virtual_machine_config = params.get("virtual_machine_config")
         self.xml_config = params.get("xml_config")
-        self.logger.info(f"LibvirtVMTask initialized with vm_name={self.vm_name}, desired_state={self.desired_state}")
+        self.volume_paths = []  # Will store volume paths for cleaning up
+        
+        # Config entity
+        self.config = config
+        if not isinstance(self.config, dict) or "default_volume_path" not in self.config:
+            self.logger.error("Config must be a dictionary containing 'default_volume_path'.")
+            raise ValueError("Config must be a dictionary containing 'default_volume_path'.")
+        
+        # If virtual_machine_config is provided, ensure volumes exist and generate XML from it
+        if self.virtual_machine_config:
+            self.logger.info(f"VirtualMachine configuration provided as JSON. Processing volumes and generating XML.")
+            # Handle storage volumes if needed
+            if "volumes" in self.virtual_machine_config:
+                self._process_storage_volumes()
+            self.xml_config = self._generate_xml_from_config()
+        elif not self.xml_config and self.desired_state != "deleted":
+            self.logger.error("Either virtual_machine_config or xml_config must be provided for non-delete operations.")
+            raise ValueError("Either virtual_machine_config or xml_config must be provided for non-delete operations.")
+            
+        self.logger.info(f"LibvirtVirtualMachineTask initialized with virtual_machine_id={self.virtual_machine_id}, desired_state={self.desired_state}")
+
+    def _get_best_machine_type(self):
+        """
+        Determine the best available 'pc-q35' machine type for KVM on x86_64.
+        Returns the highest version found in libvirt capabilities.
+        """
+        self.logger.info("Determining best machine type from libvirt capabilities.")
+        
+        # Connect to libvirt
+        try:
+            with libvirt.open("qemu:///system") as conn:
+                if conn is None:
+                    raise RuntimeError("Failed to connect to libvirt.")
+                caps_xml = conn.getCapabilities()
+        except libvirt.libvirtError as e:
+            self.logger.error(f"Libvirt connection error: {e}")
+            raise RuntimeError(f"Failed to connect to libvirt: {e}")
+
+        # Parse capabilities XML
+        try:
+            root = ET.fromstring(caps_xml)
+            
+            # Find x86_64 architecture
+            arch_elem = root.find(".//guest/arch[@name='x86_64']")
+            if not arch_elem:
+                self.logger.error("x86_64 architecture not found in libvirt capabilities.")
+                raise ValueError("x86_64 architecture not supported by the host.")
+                
+            # Check for KVM domain
+            has_kvm = any(domain.get("type") == "kvm" for domain in arch_elem.findall("domain"))
+            self.logger.debug(f"KVM domain found: {has_kvm}")
+            
+            if not has_kvm:
+                self.logger.error("KVM domain not found within x86_64 architecture.")
+                raise ValueError("KVM domain not supported for x86_64.")
+            
+            # Find all pc-q35 machine types
+            q35_machines = [
+                machine.text for machine in arch_elem.findall("machine")
+                if machine.text and machine.text.startswith("pc-q35-")
+            ]
+            
+            if not q35_machines:
+                self.logger.error("No 'pc-q35' machine types available for KVM.")
+                raise ValueError("No 'pc-q35' machine types available for KVM.")
+            
+            # Use a more robust version comparison with packaging.version
+            from packaging import version
+            
+            # Filter to include only standard versioned machine types (major.minor format)
+            version_pattern = re.compile(r"^pc-q35-(\d+\.\d+)$")
+            valid_machines = [m for m in q35_machines if version_pattern.match(m)]
+            
+            if not valid_machines:
+                self.logger.warning("No standard versioned pc-q35 machine types found.")
+                # Fall back to the original list if no standard versions found
+                highest_version = max(q35_machines, key=lambda x: x.replace("pc-q35-", ""))
+            else:
+                # Sort based on actual version comparisons rather than float conversion
+                highest_version = max(
+                    valid_machines,
+                    key=lambda x: version.parse(x.replace("pc-q35-", ""))
+                )
+            
+            self.logger.info(f"Selected machine type: {highest_version}")
+            return highest_version
+            
+        except Exception as e:
+            self.logger.error(f"Error parsing libvirt capabilities: {e}")
+            raise RuntimeError(f"Failed to determine best machine type: {e}")
+
+
+
+    def _process_storage_volumes(self):
+        """
+        Process storage volumes defined in virtual_machine_config.
+        Create any volumes that don't exist.
+        Keep track of volume paths for potential cleanup during delete.
+        """
+        self.logger.info("Processing storage volumes")
+        for volume in self.virtual_machine_config["volumes"]:
+            if "id" not in volume:
+                self.logger.error("Volume configuration missing required 'id' attribute.")
+                raise ValueError("Volume configuration missing required 'id' attribute.")
+            
+            volume_id = volume["id"]
+            format_type = volume.get("format", "qcow2")  # Default qcow2
+            path = self._get_volume_path(volume_id, format_type)
+            self.volume_paths.append(path)
+            
+            # Check if volume exists, create if it doesn't
+            if not self._check_volume_exists(path):
+                # Get optional volume parameters with defaults
+                size_gb = volume.get("size_gb", 10)  # Default 10GB
+                backing_file = volume.get("backing_file", None)
+                
+                self._create_volume(path, size_gb, format_type, backing_file)
+
+    def _get_volume_path(self, volume_id, format_type):
+        """
+        Construct the full path for a volume using the default volume path and volume ID.
+        
+        Args:
+            volume_id (str): The ID of the volume.
+            format_type (str): The format of the volume (e.g., 'qcow2').
+            
+        Returns:
+            str: The full path to the volume.
+        """
+        if format_type == "qcow2":
+            return os.path.join(self.config["default_volume_path"], f"{volume_id}.qcow2")
+        else:
+            return os.path.join(self.config["default_volume_path"], volume_id)
+
+    def _check_volume_exists(self, path):
+        """
+        Check if a storage volume exists at the specified path.
+        
+        Args:
+            path (str): Path to the storage volume.
+            
+        Returns:
+            bool: True if volume exists, False otherwise.
+        """
+        self.logger.info(f"Checking if volume exists at path: {path}")
+        return os.path.exists(path)
+
+    def _create_volume(self, path, size_gb=10, format_type="qcow2", backing_file=None):
+        """
+        Create a new storage volume.
+        
+        Args:
+            path (str): Path to the volume.
+            size_gb (int): Size of the volume in GB.
+            format_type (str): Format of the volume (qcow2, raw, etc.)
+            backing_file (str): Optional path to a backing file for thin provisioning.
+            
+        Returns:
+            bool: True if creation successful, raises exception otherwise.
+        """
+        self.logger.info(f"Creating volume at path: {path} with size: {size_gb}GB, format: {format_type}")
+        
+        # Create directory if it doesn't exist
+        directory = os.path.dirname(path)
+        if not os.path.exists(directory):
+            self.logger.info(f"Creating directory: {directory}")
+            os.makedirs(directory, exist_ok=True)
+        
+        # Build qemu-img command
+        command = ["qemu-img", "create"]
+        command.extend(["-f", format_type])
+        
+        # Add backing file if specified (for thin provisioning)
+        if backing_file:
+            if not os.path.exists(backing_file):
+                raise ValueError(f"Backing file does not exist: {backing_file}")
+            command.extend(["-b", backing_file])
+        
+        command.extend([path, f"{size_gb}G"])
+        
+        # Execute command
+        try:
+            self.logger.info(f"Executing command: {' '.join(command)}")
+            result = subprocess.run(command, check=True, capture_output=True, text=True)
+            self.logger.info(f"Volume created successfully: {path}")
+            return True
+        except subprocess.CalledProcessError as e:
+            self.logger.error(f"Failed to create volume: {e}")
+            self.logger.error(f"Command output: {e.stdout}")
+            self.logger.error(f"Command error: {e.stderr}")
+            raise RuntimeError(f"Failed to create volume: {e}")
+
+    def _delete_volume(self, path):
+        """
+        Delete a storage volume.
+        
+        Args:
+            path (str): Path to the volume to delete.
+            
+        Returns:
+            bool: True if deletion successful or file doesn't exist, False otherwise.
+        """
+        if not path:
+            return False
+            
+        self.logger.info(f"Deleting volume at path: {path}")
+        
+        if not os.path.exists(path):
+            self.logger.info(f"Volume doesn't exist, nothing to delete: {path}")
+            return True
+            
+        try:
+            os.remove(path)
+            self.logger.info(f"Volume deleted successfully: {path}")
+            return True
+        except Exception as e:
+            self.logger.error(f"Failed to delete volume {path}: {e}")
+            return False
+
+    def _generate_xml_from_config(self):
+        """
+        Generate libvirt XML configuration from virtual_machine_config dictionary.
+        
+        Args:
+            virtual_machine_id (str): A unique identifier for the VirtualMachine to be embedded in the XML.
+        
+        Returns:
+            str: XML configuration for the VirtualMachine.
+        """
+        self.logger.debug(f"Inside _generate_xml_from_config for {self.virtual_machine_id}")
+        try:
+            # Validate virtual_machine_config structure
+            required_config = ["memory", "vcpu"]
+            for key in required_config:
+                if key not in self.virtual_machine_config:
+                    raise ValueError(f"Missing required VirtualMachine configuration parameter: {key}")
+            
+            # Create root domain element
+            domain = ET.Element("domain")
+            domain.set("type", "kvm")  # Default to KVirtualMachine, could be made configurable
+            domain.set("xmlns:custom", "http://example.com/xmlns/libvirt/custom")
+
+            # Basic VirtualMachine information
+            ET.SubElement(domain, "name").text = self.virtual_machine_id
+            self.logger.debug(f"Set name")
+            # Memory configuration (in KiB)
+            memory = int(self.virtual_machine_config["memory"]) * 1024  # Convert MB to KiB
+            ET.SubElement(domain, "memory", unit="KiB").text = str(memory)
+            ET.SubElement(domain, "currentMemory", unit="KiB").text = str(memory)
+            
+            # CPU configuration
+            ET.SubElement(domain, "vcpu", placement="static").text = str(self.virtual_machine_config["vcpu"])
+
+            # OS configuration
+            os_elem = ET.SubElement(domain, "os")
+            type_elem = ET.SubElement(os_elem, "type", arch="x86_64")
+            type_elem.text = "hvm"
+            
+            # Dynamically set machine type
+            best_machine = self._get_best_machine_type()
+            type_elem.set("machine", best_machine)
+
+
+            # Boot options if specified
+            if "boot_devices" in self.virtual_machine_config:
+                for device in self.virtual_machine_config["boot_devices"]:
+                    ET.SubElement(os, "boot", dev=device)
+            
+            # Features
+            features = ET.SubElement(domain, "features")
+            ET.SubElement(features, "acpi")
+            ET.SubElement(features, "apic")
+            ET.SubElement(features, "vmport", state="off")
+            
+            # Devices section
+            devices = ET.SubElement(domain, "devices")
+            
+            # Emulator
+            ET.SubElement(devices, "emulator").text = "/usr/bin/qemu-system-x86_64"
+            
+            # Add disks/volumes
+            if "volumes" in self.virtual_machine_config:
+                for idx, volume in enumerate(self.virtual_machine_config["volumes"]):
+                    disk = ET.SubElement(devices, "disk", type="file", device="disk")
+                    ET.SubElement(disk, "driver", name="qemu", type="qcow2")
+                    
+                    # Construct the volume path dynamically
+                    volume_id = volume["id"]
+                    format_type = volume.get("format", "qcow2")  # Default qcow2
+                    path = self._get_volume_path(volume_id, format_type)
+                    ET.SubElement(disk, "source", file=path)
+                    
+                    # Determine device name based on index
+                    device_name = f"vd{chr(97 + idx)}"  # vda, vdb, etc.
+                    ET.SubElement(disk, "target", dev=device_name, bus="virtio")
+            
+            # Add network interfaces
+            if "networks" in self.virtual_machine_config:
+                for network in self.virtual_machine_config["networks"]:
+                    interface = ET.SubElement(devices, "interface", type="network")
+                    ET.SubElement(interface, "source", network=network["id"])
+                    
+                    if "mac_address" in network:
+                        ET.SubElement(interface, "mac", address=network["mac_address"])
+                    
+                    ET.SubElement(interface, "model", type="virtio")
+            
+            # Add graphics if specified
+            if "graphics" in self.virtual_machine_config:
+                graphics_config = self.virtual_machine_config["graphics"]
+                graphics = ET.SubElement(devices, "graphics", type=graphics_config.get("type", "vnc"))
+                
+                for attr, value in graphics_config.items():
+                    if attr != "type":
+                        graphics.set(attr, str(value))
+            
+            # Add metadata section to store VirtualMachine ID and VirtualMachine name
+            metadata = ET.SubElement(domain, "metadata")
+            custom_metadata = ET.SubElement(metadata, "custom:metadata", xmlns_custom="http://example.com/xmlns/libvirt/custom")
+            ET.SubElement(custom_metadata, "custom:virtual_machine_id").text = self.virtual_machine_id
+            ET.SubElement(custom_metadata, "custom:virtual_machine_name").text = self.virtual_machine_id
+
+            # Convert to pretty XML string
+            rough_string = ET.tostring(domain, 'utf-8')
+            reparsed = minidom.parseString(rough_string)
+            xml_config = reparsed.toprettyxml(indent="  ")
+            self.logger.info(f"Generated XML configuration for VirtualMachine '{self.virtual_machine_id}' with ID '{self.virtual_machine_id}'.")
+            return xml_config
+            
+        except Exception as e:
+            self.logger.error(f"Error generating XML configuration: {e}")
+            raise ValueError(f"Failed to generate XML configuration: {e}")
+        
 
     def execute(self):
         """
-        Execute the task: Manage VM state and optionally perform actions.
+        Execute the task: Manage VirtualMachine state and optionally perform actions.
 
         Returns:
             dict: Response payload indicating success or failure.
@@ -64,87 +414,297 @@ class LibvirtVMTask:
             if conn is None:
                 raise RuntimeError("Failed to open connection to libvirt.")
 
-            # Lookup VM by name
+            # Extract volume paths from VirtualMachine if it already exists
+            # This is needed for deletion when the task is called without virtual_machine_config
+            if not self.volume_paths:
+                try:
+                    domain = conn.lookupByName(self.virtual_machine_id)
+                    self._extract_volume_paths_from_domain(domain)
+                except libvirt.libvirtError:
+                    # VirtualMachine doesn't exist, so no volumes to extract
+                    pass
+
+            # Lookup VirtualMachine by name
+            self.logger.info("Looking up VirtualMachine Name")
             try:
-                domain = conn.lookupByName(self.vm_name)
-                self.logger.info(f"VM '{self.vm_name}' exists. Managing state.")
-            except libvirt.libvirtError:
+                # Get all domains (VirtualMachines) managed by the libvirt connection
+                all_domains = conn.listAllDomains(0)  # 0 means no flags, return all domains
+                self.logger.info(f"Retrieved {len(all_domains)} VirtualMachines from libvirt.")
+
+                # Search for the VirtualMachine with the given name
                 domain = None
-                self.logger.info(f"VM '{self.vm_name}' does not exist.")
+                for dom in all_domains:
+                    if dom.name() == self.virtual_machine_id:
+                        domain = dom
+                        self.logger.info(f"VirtualMachine '{self.virtual_machine_id}' exists. Managing state.")
+                        break
+
+                if domain is None:
+                    self.logger.info(f"VirtualMachine '{self.virtual_machine_id}' does not exist.")
+            except libvirt.libvirtError as e:
+                self.logger.error(f"Error retrieving domains from libvirt: {e}")
+                raise e
 
             # Handle desired state
             if self.desired_state == "deleted":
                 if domain:
-                    self.logger.info(f"Deleting VM '{self.vm_name}'.")
+                    self.logger.info(f"Deleting VirtualMachine '{self.virtual_machine_id}'.")
                     if domain.isActive():
-                        domain.destroy()  # Stop the VM if running
+                        domain.destroy()  # Stop the VirtualMachine if running
                     domain.undefine()  # Remove its definition
-                    response_payload["response"]["status"] = "VM deleted"
+                    
+                    # Delete associated storage volumes
+                    self._delete_storage_volumes()
+                    
+                    response_payload["response"]["status"] = "VirtualMachine deleted"
                 else:
-                    self.logger.info(f"VM '{self.vm_name}' is already deleted.")
-                    response_payload["response"]["status"] = "VM already deleted"
+                    self.logger.info(f"VirtualMachine '{self.virtual_machine_id}' is already deleted.")
+                    response_payload["response"]["status"] = "VirtualMachine already deleted"
                 response_payload["success"] = True
 
             elif self.desired_state in ["running", "stopped"]:
                 if domain:
-                    self.logger.info(f"VM '{self.vm_name}' exists. Checking state.")
+                    # Check if VirtualMachine config has changed
+                    if self.xml_config:
+                        current_xml = domain.XMLDesc()
+                        if self._is_xml_config_different(current_xml, self.xml_config):
+                            self.logger.info(f"VirtualMachine configuration has changed. Updating VirtualMachine '{self.virtual_machine_id}'.")
+                            # Need to recreate the VirtualMachine with new configuration
+                            if domain.isActive():
+                                domain.destroy()  # Stop the VirtualMachine if running
+                            domain.undefine()  # Remove its definition
+                            domain = conn.defineXML(self.xml_config)  # Create with new config
+                    
+                    self.logger.info(f"VirtualMachine '{self.virtual_machine_id}' exists. Checking state.")
                     if self.desired_state == "running":
-                        self.logger.info(f"VM desired state is running, current state is {domain.isActive()}")
+                        self.logger.info(f"VirtualMachine desired state is running, current state is {domain.isActive()}")
                         if not domain.isActive():
-                            self.logger.info(f"Starting VM '{self.vm_name}'.")
-                            domain.create()  # Start the VM
-                        response_payload["response"]["status"] = "VM running"
+                            self.logger.info(f"Starting VirtualMachine '{self.virtual_machine_id}'.")
+                            domain.create()  # Start the VirtualMachine
+                        response_payload["response"]["status"] = "VirtualMachine running"
                     else:  # desired_state == "stopped"
-                        self.logger.info(f"VM desired state is stopped, current state is {domain.isActive()}")
+                        self.logger.info(f"VirtualMachine desired state is stopped, current state is {domain.isActive()}")
                         if domain.isActive():
-                            self.logger.info(f"Stopping VM '{self.vm_name}'.")
-                            domain.destroy()  # Stop the VM
-                        response_payload["response"]["status"] = "VM stopped"
+                            self.logger.info(f"Stopping VirtualMachine '{self.virtual_machine_id}'.")
+                            domain.destroy()  # Stop the VirtualMachine
+                        response_payload["response"]["status"] = "VirtualMachine stopped"
                 else:
                     if not self.xml_config:
-                        raise ValueError(f"XML configuration is required to create VM '{self.vm_name}'.")
-                    self.logger.info(f"Creating VM '{self.vm_name}' with the provided configuration.")
+                        raise ValueError(f"XML configuration is required to create VirtualMachine '{self.virtual_machine_id}'.")
+                    self.logger.info(f"Creating VirtualMachine '{self.virtual_machine_id}' with the provided configuration.")
                     domain = conn.defineXML(self.xml_config)
                     if self.desired_state == "running":
-                        domain.create()  # Start the VM
-                        response_payload["response"]["status"] = "VM created and running"
+                        domain.create()  # Start the VirtualMachine
+                        response_payload["response"]["status"] = "VirtualMachine created and running"
                     else:
-                        response_payload["response"]["status"] = "VM created and stopped"
+                        response_payload["response"]["status"] = "VirtualMachine created and stopped"
                 response_payload["success"] = True
 
             # Handle optional actions
             if self.action:
                 if not domain:
-                    raise RuntimeError(f"Cannot perform action '{self.action}' on a non-existent VM.")
+                    raise RuntimeError(f"Cannot perform action '{self.action}' on a non-existent VirtualMachine.")
 
                 if self.action == "start":
                     if not domain.isActive():
-                        self.logger.info(f"Starting VM '{self.vm_name}'.")
+                        self.logger.info(f"Starting VirtualMachine '{self.virtual_machine_id}'.")
                         domain.create()
-                    response_payload["response"]["action_status"] = "VM started"
+                    response_payload["response"]["action_status"] = "VirtualMachine started"
                 elif self.action == "stop":
                     if domain.isActive():
-                        self.logger.info(f"Stopping VM '{self.vm_name}'.")
+                        self.logger.info(f"Stopping VirtualMachine '{self.virtual_machine_id}'.")
                         domain.destroy()
-                    response_payload["response"]["action_status"] = "VM stopped"
+                    response_payload["response"]["action_status"] = "VirtualMachine stopped"
                 elif self.action == "reboot":
                     if domain.isActive():
-                        self.logger.info(f"Rebooting VM '{self.vm_name}'.")
+                        self.logger.info(f"Rebooting VirtualMachine '{self.virtual_machine_id}'.")
                         domain.reboot()
-                        response_payload["response"]["action_status"] = "VM rebooted"
+                        response_payload["response"]["action_status"] = "VirtualMachine rebooted"
                     else:
-                        raise RuntimeError("Cannot reboot a stopped VM.")
+                        raise RuntimeError("Cannot reboot a stopped VirtualMachine.")
                 else:
                     raise ValueError(f"Unsupported action: {self.action}")
-                self.logger.info(f"Action '{self.action}' completed successfully for VM '{self.vm_name}'.")
+                self.logger.info(f"Action '{self.action}' completed successfully for VirtualMachine '{self.virtual_machine_id}'.")
 
         except Exception as e:
-            self.logger.error(f"Error executing LibvirtVMTask: {e}")
+            self.logger.error(f"Error executing LibvirtVirtualMachineTask: {e}")
             response_payload["response"]["error"] = str(e)
 
         finally:
             if 'conn' in locals() and conn:
                 conn.close()
 
-        self.logger.info(f"LibvirtVMTask response: {response_payload}")
+        self.logger.info(f"LibvirtVirtualMachineTask response: {response_payload}")
         return response_payload
+
+    def _extract_volume_paths_from_domain(self, domain):
+        """
+        Extract volume paths from an existing domain.
+        This is useful when deleting a VirtualMachine without having its original configuration.
+        
+        Args:
+            domain: libvirt domain object
+        """
+        self.logger.info(f"Extracting volume paths from existing domain '{self.virtual_machine_id}'")
+        try:
+            xml_desc = domain.XMLDesc()
+            root = ET.fromstring(xml_desc)
+            
+            # Find all disk elements that are actually files
+            for disk in root.findall(".//devices/disk[@type='file']"):
+                source = disk.find("source")
+                if source is not None and "file" in source.attrib:
+                    file_path = source.attrib["file"]
+                    self.logger.info(f"Found volume path: {file_path}")
+                    if file_path not in self.volume_paths:
+                        self.volume_paths.append(file_path)
+        except Exception as e:
+            self.logger.error(f"Error extracting volume paths: {e}")
+
+    def _delete_storage_volumes(self):
+        """
+        Delete all associated storage volumes.
+        """
+        self.logger.info(f"Deleting storage volumes for VirtualMachine '{self.virtual_machine_id}'")
+        for path in self.volume_paths:
+            self._delete_volume(path)
+
+    def _is_xml_config_different(self, current_xml, new_xml):
+        """
+        Compare current VirtualMachine XML with the new XML configuration to determine if update is needed.
+        
+        Args:
+            current_xml (str): Current XML configuration of the VirtualMachine.
+            new_xml (str): New XML configuration to be applied.
+            
+        Returns:
+            bool: True if configurations are different, False otherwise.
+        """
+        # Use difflib to determine differences between XML configurations
+        # Note: This is a simple comparison and might not catch all semantic differences
+        diff = list(difflib.unified_diff(
+            current_xml.splitlines(),
+            new_xml.splitlines(),
+            n=0
+        ))
+        
+        # Return True if there are differences
+        return len(diff) > 0
+    
+
+
+
+    # # Example usage
+    # json_input = '''
+    # {
+    #     "url": "https://example.com/path/to/image.jpg.gz",
+    #     "md5": "d41d8cd98f00b204e9800998ecf8427e",
+    #     "compressed": true
+    # }
+    # '''
+    # logger.info("Starting process_image function...")
+    # result = process_image(json_input)
+    # logger.info(f"Result: {result}")
+    def _process_image(self, json_string):
+        # Parse the JSON input
+        self.logger.info("Processing image - Parsing JSON input...")
+        try:
+            data = json.loads(json_string)
+            url = data['url']
+            expected_md5 = data['md5']
+            is_compressed = data.get('compressed', False)  # Default to False if not provided
+            self.logger.debug(f"URL: {url}, Expected MD5: {expected_md5}, Compressed: {is_compressed}")
+        except json.JSONDecodeError as e:
+            self.logger.error(f"Invalid JSON input: {e}")
+            return f"Error: Invalid JSON input. {e}"
+        except KeyError as e:
+            self.logger.error(f"Missing required key in JSON: {e}")
+            return f"Error: Missing required key in JSON. {e}"
+
+        # Extract the final file name from the URL
+        file_name = os.path.basename(url)
+        if is_compressed:
+            file_name = file_name.rstrip('.gz')  # Remove .gz extension if compressed
+        md5_file_name = f"{file_name}.md5"
+        self.logger.debug(f"File name: {file_name}, MD5 file name: {md5_file_name}")
+
+        # Check if the image file exists locally
+        if os.path.exists(file_name):
+            self.logger.info(f"Image file '{file_name}' exists locally.")
+            # Check if the MD5 file exists
+            if os.path.exists(md5_file_name):
+                self.logger.info(f"MD5 file '{md5_file_name}' exists.")
+                with open(md5_file_name, 'r') as md5_file:
+                    local_md5 = md5_file.read().strip()
+                self.logger.debug(f"Local MD5: {local_md5}, Expected MD5: {expected_md5}")
+
+                # Compare the local MD5 with the expected MD5
+                if local_md5 == expected_md5:
+                    self.logger.info("MD5 matches. No further action needed.")
+                    return True
+                else:
+                    self.logger.warning("MD5 mismatch. Deleting local files and downloading again.")
+                    os.remove(file_name)
+                    os.remove(md5_file_name)
+            else:
+                self.logger.warning(f"MD5 file '{md5_file_name}' missing. Deleting local image and downloading again.")
+                os.remove(file_name)
+        else:
+            self.logger.info(f"Image file '{file_name}' does not exist locally. Downloading...")
+
+        # Download the image from the URL using streaming
+        temp_file_name = f"{file_name}.tmp"
+        try:
+            self.logger.info(f"Downloading image from URL: {url}")
+            with requests.get(url, stream=True) as response:
+                response.raise_for_status()  # Raise an error for bad status codes
+                with open(temp_file_name, 'wb') as temp_file:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        temp_file.write(chunk)
+                self.logger.info(f"Download complete. Temporary file saved as '{temp_file_name}'.")
+
+            # If the image is compressed, uncompress it
+            if is_compressed:
+                self.logger.info(f"Uncompressing the image '{temp_file_name}'...")
+                with gzip.open(temp_file_name, 'rb') as f_in:
+                    with open(file_name, 'wb') as f_out:
+                        shutil.copyfileobj(f_in, f_out)
+                self.logger.info(f"Uncompressed image saved as '{file_name}'.")
+                os.remove(temp_file_name)  # Delete the temporary compressed file
+            else:
+                os.rename(temp_file_name, file_name)  # Rename the temporary file to the final name
+                self.logger.info(f"Renamed temporary file '{temp_file_name}' to '{file_name}'.")
+
+        except requests.exceptions.RequestException as e:
+            self.logger.error(f"Failed to download the image. Error: {e}")
+            if os.path.exists(temp_file_name):
+                os.remove(temp_file_name)
+            return f"Error: Failed to download the image. {e}"
+
+        # Calculate the MD5 hash of the downloaded image
+        self.logger.info(f"Calculating MD5 hash for '{file_name}'...")
+        md5_hash = hashlib.md5()
+        with open(file_name, 'rb') as image_file:
+            for chunk in iter(lambda: image_file.read(4096), b""):
+                md5_hash.update(chunk)
+        actual_md5 = md5_hash.hexdigest()
+        self.logger.debug(f"Calculated MD5: {actual_md5}, Expected MD5: {expected_md5}")
+
+        # Compare the calculated MD5 with the expected MD5
+        if actual_md5 == expected_md5:
+            self.logger.info("MD5 matches. Saving MD5 hash to file...")
+            with open(md5_file_name, 'w') as md5_file:
+                md5_file.write(actual_md5)
+            self.logger.info(f"MD5 hash saved to '{md5_file_name}'.")
+            return True
+        else:
+            self.logger.error(f"MD5 mismatch. Expected {expected_md5}, got {actual_md5}.")
+            # Delete the downloaded image and MD5 file if they exist
+            if os.path.exists(file_name):
+                os.remove(file_name)
+                self.logger.info(f"Deleted file '{file_name}'.")
+            if os.path.exists(md5_file_name):
+                os.remove(md5_file_name)
+                self.logger.info(f"Deleted file '{md5_file_name}'.")
+            return f"Error: MD5 mismatch. Expected {expected_md5}, got {actual_md5}."
+        

workload_states.md

@@ -0,0 +1,8 @@
+States
+
+
+pending-allocation - Workload creeated in DB, has not been dispatched to the worker yet
+pending-allocated - Task assigned to a worker and dispatched to the websocket server
+
+running - Worker has called back the workload is running
+deleted - deleted