diff --git a/.gitignore b/.gitignore index 30654f1..69c941c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ instance/* logs/* .venv/ mariadb_data/ -.env \ No newline at end of file +.env +.aider* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..690a5b1 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "app/static/novnc"] + path = app/static/novnc + url = https://github.com/novnc/noVNC.git diff --git a/api_client/client.py b/api_client/client.py index 0637923..a84477e 100644 --- a/api_client/client.py +++ b/api_client/client.py @@ -1,11 +1,9 @@ -# import streamlit as st import requests -# This API client is used by streamlit and by the websocket server class IaaSClient: def __init__(self, api_key, base_url="http://localhost:5000/api"): self.base_url = base_url - self.api_key=api_key + self.api_key = api_key def _make_request(self, method, endpoint, data=None): headers = { @@ -24,80 +22,245 @@ class IaaSClient: print(f"API Error: {str(e)}") return None + # Universe CRUD def get_universes(self): return self._make_request('GET', 'universes') + def get_universe(self, universe_id): + return self._make_request('GET', f'universes/{universe_id}') + def create_universe(self, name, description): return self._make_request('POST', 'universes', {'name': name, 'description': description}) + def update_universe(self, universe_id, data): + return self._make_request('PUT', f'universes/{universe_id}', data) + + def delete_universe(self, universe_id): + return self._make_request('DELETE', f'universes/{universe_id}') + + # Project CRUD def get_projects(self): return self._make_request('GET', 'projects') + def get_project(self, project_id): + return self._make_request('GET', f'projects/{project_id}') + def create_project(self, name, universe_id): return self._make_request('POST', 'projects', {'name': name, 'universe_id': universe_id}) - def get_vdcs(self): - return self._make_request('GET', 'vdcs') + def update_project(self, project_id, data): + return self._make_request('PUT', f'projects/{project_id}', data) - def create_vdc(self, name, project_id): - return self._make_request('POST', 'vdcs', {'name': name, 'project_id': project_id}) - + def delete_project(self, project_id): + return self._make_request('DELETE', f'projects/{project_id}') + + # virtual_data_centers CRUD + def get_virtual_data_centers(self): + return self._make_request('GET', 'virtual_data_centers') + + def get_virtual_data_center(self, virtual_data_center_id): + return self._make_request('GET', f'virtual_data_centers/{virtual_data_center_id}') + + def create_virtual_data_center(self, name, project_id): + return self._make_request('POST', 'virtual_data_centers', {'name': name, 'project_id': project_id}) + + def update_virtual_data_center(self, virtual_data_center_id, data): + return self._make_request('PUT', f'virtual_data_centers/{virtual_data_center_id}', data) + + def delete_virtual_data_center(self, virtual_data_center_id): + return self._make_request('DELETE', f'virtual_data_centers/{virtual_data_center_id}') + + # Network CRUD def get_networks(self): return self._make_request('GET', 'networks') - def create_network(self, name, vdc_id): - return self._make_request('POST', 'networks', {'name': name, 'vdc_id': vdc_id}) + def get_network(self, network_id): + return self._make_request('GET', f'networks/{network_id}') + + def create_network(self, name, virtual_data_center_id): + return self._make_request('POST', 'networks', {'name': name, 'virtual_data_center_id': virtual_data_center_id}) + + def update_network(self, network_id, data): + return self._make_request('PUT', f'networks/{network_id}', data) + + def delete_network(self, network_id): + return self._make_request('DELETE', f'networks/{network_id}') + + # Network Port CRUD + def get_network_ports(self): + return self._make_request('GET', 'network_ports') + + def get_network_port(self, network_port_id): + return self._make_request('GET', f'network_ports/{network_port_id}') + + def create_network_port(self, data): + return self._make_request('POST', 'network_ports', data) + + def update_network_port(self, network_port_id, data): + return self._make_request('PUT', f'network_ports/{network_port_id}', data) + + def delete_network_port(self, network_port_id): + return self._make_request('DELETE', f'network_ports/{network_port_id}') def get_network_ports_by_network(self,network_id): return self._make_request('GET', f'network_ports/by_network/{network_id}') + + # Region CRUD def get_regions(self): return self._make_request('GET', 'regions') - + + def get_region(self, region_id): + return self._make_request('GET', f'regions/{region_id}') + def create_region(self, name, description): return self._make_request('POST', 'regions', {'name': name, 'description': description}) - - def get_region_access(self, region_id): - return self._make_request('GET', f'region_access/by_region/{region_id}') + def update_region(self, region_id, data): + return self._make_request('PUT', f'regions/{region_id}', data) + + def delete_region(self, region_id): + return self._make_request('DELETE', f'regions/{region_id}') + + # Region Access CRUD + def get_region_accesses(self): + return self._make_request('GET', 'region_access') + + def get_region_access(self, region_access_id): + return self._make_request('GET', f'region_access/{region_access_id}') + + def create_region_access(self, data): + return self._make_request('POST', 'region_access', data) + + def update_region_access(self, region_access_id, data): + return self._make_request('PUT', f'region_access/{region_access_id}', data) + + def delete_region_access(self, region_access_id): + return self._make_request('DELETE', f'region_access/{region_access_id}') + + # Workload Host CRUD def get_workload_hosts(self): return self._make_request('GET', 'workload_hosts') - def get_workload_host(self,workload_host_id): + def get_workload_host(self, workload_host_id): return self._make_request('GET', f'workload_hosts/{workload_host_id}') - + + def create_workload_host(self, data): + return self._make_request('POST', 'workload_hosts', data) + + def update_workload_host(self, workload_host_id, data): + return self._make_request('PUT', f'workload_hosts/{workload_host_id}', data) + + def delete_workload_host(self, workload_host_id): + return self._make_request('DELETE', f'workload_hosts/{workload_host_id}') + + # Image CRUD def get_images(self): return self._make_request('GET', 'images') - - def get_image(self,image_id): + + def get_image(self, image_id): return self._make_request('GET', f'images/{image_id}') - - def edit_image(self,image_id,data): - return self._make_request('PUT', f'images/{image_id}',data) def create_image(self, data): - # Make the API request - return self._make_request('POST', 'images', data) - + print(":creating image") + return self._make_request('POST', 'images', data) + def update_image(self, image_id, data): + return self._make_request('PUT', f'images/{image_id}', data) + def delete_image(self, image_id): + return self._make_request('DELETE', f'images/{image_id}') + # Container CRUD def get_containers(self): return self._make_request('GET', 'workloads/containers') - - def get_container(self,container_id): + + def get_container(self, container_id): return self._make_request('GET', f'workloads/containers/{container_id}') - - def edit_container(self,container_id,data): - return self._make_request('PUT', f'workloads/containers/{container_id}',data) def create_container(self, data): - # Make the API request - return self._make_request('POST', 'workloads/containers', data) + return self._make_request('POST', 'workloads/containers', data) + + def update_container(self, container_id, data): + return self._make_request('PUT', f'workloads/containers/{container_id}', data) def delete_container(self, container_id): - # Make the API request - return self._make_request('DELETE', f'workloads/containers/{container_id}') + return self._make_request('DELETE', f'workloads/containers/{container_id}') + + # Virtual Machine CRUD + def get_virtual_machines(self): + return self._make_request('GET', 'workloads/virtual_machines') + + def get_virtual_machine(self, virtual_machine_id): + return self._make_request('GET', f'workloads/virtual_machines/{virtual_machine_id}') + + def create_virtual_machine(self, data): + return self._make_request('POST', 'workloads/virtual_machines', data) + + def update_virtual_machine(self, virtual_machine_id, data): + return self._make_request('PUT', f'workloads/virtual_machines/{virtual_machine_id}', data) + + def delete_virtual_machine(self, virtual_machine_id): + return self._make_request('DELETE', f'workloads/virtual_machines/{virtual_machine_id}') + + # Workload CRUD + def get_workloads(self): + return self._make_request('GET', 'workloads') + + def get_workload(self, workload_id): + return self._make_request('GET', f'workloads/{workload_id}') + + def create_workload(self, data): + return self._make_request('POST', 'workloads', data) + + def update_workload(self, workload_id, data): + return self._make_request('PUT', f'workloads/{workload_id}', data) + + def delete_workload(self, workload_id): + return self._make_request('DELETE', f'workloads/{workload_id}') + + + # Workload Host Resources + def get_pooled_resources_for_host(self, host_id): + """ + Fetch pooled resources for a specific workload host. + """ + return self._make_request('GET', f'workload-hosts/{host_id}/pooled-resources') + + def get_fixed_resources_for_host(self, host_id): + """ + Fetch fixed resources for a specific workload host. + """ + return self._make_request('GET', f'workload-hosts/{host_id}/fixed-resources') + + def get_ovs_bridges_for_host(self, host_id): + """ + Fetch OVS bridges for a specific workload host. + """ + return self._make_request('GET', f'workload-hosts/{host_id}/ovs-bridges') - \ No newline at end of file + + + # Workloads for Host + def get_workloads_for_host(self, host_id): + """ + Fetch all workloads (containers and virtual machines) for a specific host. + """ + # Fetch all containers and virtual machines + containers = self.get_containers() or [] + virtual_machines = self.get_virtual_machines() or [] + + # Filter workloads by host_id + workloads = [] + for container in containers: + if container.get('workload_host_id') == host_id: + container['workload_type'] = 'container' # Add workload type + workloads.append(container) + + for vm in virtual_machines: + if vm.get('workload_host_id') == host_id: + vm['workload_type'] = 'virtual_machine' # Add workload type + workloads.append(vm) + + return workloads \ No newline at end of file diff --git a/app/controller/__init__.py b/app/controller/__init__.py index 4e9678b..6e4a20f 100644 --- a/app/controller/__init__.py +++ b/app/controller/__init__.py @@ -19,4 +19,6 @@ from .api.universe_routes import * from .api.workload_host_routes import * from .api.region_access_routes import * from .api.workload_container_routes import * -from .api.image_routes import * \ No newline at end of file +from .api.image_routes import * +from .api.workload_vm_routes import * +from .api.virtual_data_center_routes import * \ No newline at end of file diff --git a/app/controller/api/api_routes.py b/app/controller/api/api_routes.py index 4238573..e4b1097 100644 --- a/app/controller/api/api_routes.py +++ b/app/controller/api/api_routes.py @@ -1,6 +1,6 @@ from flask import request, jsonify from app.models.models import ( - Universe, Project, VirtualDataCenter, Workload, Network, Region, RegionAccess, Volume, Image, + Universe, Project, VirtualDataCenter, Workload, Region, RegionAccess, Volume, Image, WorkloadHost, WorkloadHostFixedResource, WorkloadHostPooledResource, Label, User ) from app import app, db @@ -57,8 +57,7 @@ def create_routes(model, endpoint): # Register API routes for each model # create_routes(Universe, 'universes') # create_routes(Project, 'projects') -create_routes(VirtualDataCenter, 'vdcs') -# create_routes(Network, 'networks') +# create_routes(VirtualDataCenter, 'vdcs') # create_routes(Region, 'regions') create_routes(Volume, 'volumes') # create_routes(Image, 'images') diff --git a/app/controller/api/image_routes.py b/app/controller/api/image_routes.py index 1655c49..cea702c 100644 --- a/app/controller/api/image_routes.py +++ b/app/controller/api/image_routes.py @@ -41,6 +41,7 @@ def validate_image_data(data, is_update=False): 'location_type': str, 'os_family': str, 'os_version': str, + 'checksum': str, } for field, field_type in optional_fields.items(): if field in data and not isinstance(data[field], field_type): @@ -65,7 +66,8 @@ def add_image(): size=validated_data['size'], location_type=validated_data.get('location_type'), os_family=validated_data.get('os_family'), - os_version=validated_data.get('os_version') + os_version=validated_data.get('os_version'), + checksum=validated_data.get('checksum') ) db.session.add(instance) db.session.commit() diff --git a/app/controller/api/network_routes.py b/app/controller/api/network_routes.py index 117a2be..2bff36b 100644 --- a/app/controller/api/network_routes.py +++ b/app/controller/api/network_routes.py @@ -1,6 +1,6 @@ from flask import request, jsonify, abort from app import app, db, logger -from app.models.models import Network, NetworkPort +from app.models.network import Network, NetworkPort from datetime import datetime from app.controller import api_bp import uuid @@ -102,7 +102,7 @@ def edit_network(network_id): @api_bp.route('/networks/', methods=['GET']) def get_network(network_id): - network = Network.query.get_or_404(network_id) + network = Network.query.get_or_404(uuid.UUID(network_id)) return jsonify(network.to_json()) @api_bp.route('/networks/', methods=['DELETE']) diff --git a/app/controller/api/project_routes.py b/app/controller/api/project_routes.py index b3e536a..d937741 100644 --- a/app/controller/api/project_routes.py +++ b/app/controller/api/project_routes.py @@ -43,7 +43,7 @@ def edit_project(project_id): @api_bp.route('/projects/', methods=['GET']) def get_project(project_id): - project = Project.query.get_or_404(project_id) + project = Project.query.get_or_404(uuid.UUID(project_id)) return jsonify(project.to_json()) diff --git a/app/controller/api/region_routes.py b/app/controller/api/region_routes.py index c8737ae..8ff138c 100644 --- a/app/controller/api/region_routes.py +++ b/app/controller/api/region_routes.py @@ -31,9 +31,12 @@ def edit_region(region_id): # Can they see ALL the details or just the public data? @api_bp.route('/regions/') def get_region(region_id): - region = Region.query.get_or_404(region_id) - return jsonify(region.to_json()) - + region = Region.query.get_or_404(uuid.UUID(region_id)) + return jsonify({ + 'name': region.name, + 'id': region.id, + 'status': region.status, + }) @app.route('/regions/', methods=['DELETE']) diff --git a/app/controller/api/universe_routes.py b/app/controller/api/universe_routes.py index 8aa7ee2..6466671 100644 --- a/app/controller/api/universe_routes.py +++ b/app/controller/api/universe_routes.py @@ -37,7 +37,7 @@ def edit_universe(universe_id): @api_bp.route('/universes/', methods=['GET']) def get_universe(universe_id): - universe = Universe.query.get_or_404(universe_id) + universe = Universe.query.get_or_404(uuid.UUID(universe_id)) return jsonify(universe.to_json()) @api_bp.route('/universes/', methods=['DELETE']) diff --git a/app/controller/api/virtual_data_center_routes.py b/app/controller/api/virtual_data_center_routes.py new file mode 100644 index 0000000..26dfa86 --- /dev/null +++ b/app/controller/api/virtual_data_center_routes.py @@ -0,0 +1,53 @@ +from flask import request, jsonify +from app import app, db, logger +from app.models.models import VirtualDataCenter +from app.controller import api_bp +import uuid + +@api_bp.route('/virtual_data_centers', methods=['POST']) +def add_vdc(): + data = request.json + instance = VirtualDataCenter( + name=data['name'], + description=data.get('description'), + status=data.get('status'), + created_by=data.get('created_by') + ) + db.session.add(instance) + db.session.commit() + logger.debug("vdc added to DB") + return jsonify(instance.to_json()), 201 + +@api_bp.route('/virtual_data_centers/', methods=['PUT']) +def edit_vdc(virtual_data_center_id): + vdc = VirtualDataCenter.query.get_or_404(virtual_data_center_id) + data = request.json + + if 'name' in data: + vdc.name = data['name'] + if 'description' in data: + vdc.description = data['description'] + if 'status' in data: + vdc.status = data['status'] + if 'visible' in data: + vdc.visible = data['visible'] + + db.session.commit() + return jsonify(success=True) + +@api_bp.route('/virtual_data_centers/', methods=['GET']) +def get_virtual_data_center(virtual_data_center_id): + vdc = VirtualDataCenter.query.get_or_404(uuid.UUID(virtual_data_center_id)) + return jsonify(vdc.to_json()) + +@api_bp.route('/virtual_data_centers/', methods=['DELETE']) +def delete_vdc(virtual_data_center_id): + vdc = VirtualDataCenter.query.get_or_404(virtual_data_center_id) + vdc.soft_delete() + db.session.commit() + return jsonify({'message': 'vdc deleted successfully'}), 200 + +@api_bp.route('/virtual_data_centers', methods=['GET']) +def get_virtual_data_centers(): + virtual_data_centers = VirtualDataCenter.query.filter_by(deleted=False).all() + return jsonify([vdc.to_json() for vdc in virtual_data_centers]) diff --git a/app/controller/api/workload_container_routes.py b/app/controller/api/workload_container_routes.py index 50b4754..1c0d37b 100644 --- a/app/controller/api/workload_container_routes.py +++ b/app/controller/api/workload_container_routes.py @@ -90,7 +90,6 @@ def validate_payload(payload): def add_container_workload(): request_data = request.json logger.debug(request_data) - all_containers=[] # Validate incoming data validated_data = validate_payload(request_data) @@ -115,42 +114,43 @@ def add_container_workload(): # TODO - This is where we would slot in the 'placement' routine random_host = random.choice(all_workload_hosts) logger.info(f"Randomly selected host: {random_host}") - + NSController=None # Check if we need to deploy an NS Controller to this host for this VPC ID or not # TODO - We will need to update the worker code too. # - If the workder doesnt have an NS controller present for the given VPCID(Currently called a tennacny) then fail the request # Lets try and add the NSController deployment request into the same payload - ns_controller_query=Workload.query.filter_by( + existing_NS_Controller=Workload.query.filter_by( workload_host_id=random_host.id, workload_type="NSController", # Always set workload_type to "Container" vdc_id=request_vdc.id, deleted=0 - ).all() + ).first() - if len(ns_controller_query)==0: + if not existing_NS_Controller: + # No Namespace controller exists on this host, lets create one logger.warning(f"No NSController found in on host {random_host.name} for VDC {request_vdc.name}, adding it to the payload") # TODO - we need to assign a network port to the NS Controller including assigning an IP - How do we specify this in the payload? + new_NSController_name=f"NSCONTROLLER_{request_vdc.id}_{random_host.id}" new_NSController = Workload( - name=_container['container_name'], + name=new_NSController_name, workload_type="NSController", # Always set workload_type to "Container" vdc_id=request_vdc.id, status="pending-allocation", launch_params=json.dumps({ "docker_image": "busybox", - "container_name": f"NSCONTROLLER-{request_vdc.id}" + "container_name": new_NSController_name, + "command": "sleep infinite" }), workload_host_id=random_host.id ) db.session.add(new_NSController) db.session.commit() - - - NSController_launchparams=json.loads(new_NSController.launch_params) - NSController_launchparams['container_id']=new_NSController.id - all_containers.append(NSController_launchparams) - + NSController=new_NSController + else: + NSController=existing_NS_Controller + # Create the Workload instance in the database new_container = Workload( name=_container['container_name'], @@ -164,17 +164,19 @@ def add_container_workload(): db.session.commit() logger.debug(f"Container {new_container.id} workload added to DB") - #Inject the container ID so that it can be processed by the workloadHost + # Inject the container ID so that it can be processed by the workloadHost _container['container_id']=new_container.id - logger.error(_container) + _container['NSController_launchparams']=json.loads(NSController.launch_params) + _container['NSController_launchparams']['container_id']=NSController.id + + logger.debug(f"Container spec: {_container}") #Send the request off to the websocket server to have this container launched - all_containers.append(_container) payload = { "worker_id": random_host.id, "task_type": "container-create", "job_details": { "tenancyID": 3, - "containers": json.dumps(all_containers) + "container": json.dumps(_container) } } logger.debug(f"Sending payload to websocket server {payload}") @@ -185,11 +187,16 @@ def add_container_workload(): if websocket_server_response.status_code == 201: logger.info(f"Task created successfully for container {new_container.id}") + NSController.status="pending-allocated" + db.session.add(NSController) + new_container.status="pending-allocated" db.session.add(new_container) db.session.commit() else: logger.error(f"Creation request failed for container {new_container.id}") + NSController.status="pending-allocated" + db.session.add(NSController) new_container.status="failed-allocation" db.session.add(new_container) db.session.commit() @@ -274,11 +281,10 @@ def delete_container_workload(workload_id): "worker_id": _container.workload_host_id, "task_type": "container-delete", "job_details": { - "tenancyID": 3, - "containers": [ + "container": [ { "container_id": _container.id, - "deleted": "True" + "desired_state": "deleted" } ] } diff --git a/app/controller/api/workload_vm_routes.py b/app/controller/api/workload_vm_routes.py new file mode 100644 index 0000000..d82923a --- /dev/null +++ b/app/controller/api/workload_vm_routes.py @@ -0,0 +1,555 @@ +import random +from flask import json, request, jsonify, abort +import requests +from app import app, db, logger +from app.models.models import Volume, Workload, WorkloadHost, VirtualDataCenter, Image +from datetime import datetime +from app.controller import api_bp +import uuid + +import uuid +from werkzeug.exceptions import abort +from sqlalchemy import or_ + +from app.models.network import Network + +websocket_server_url = "http://127.0.0.1:6000/api/create_task" + +def validate_port_entry(port_entry): + """ + Validate a single port entry in the open_ports list. + Valid formats include: + - protocol/port (e.g., "tcp/80") + - protocol/port-range (e.g., "tcp/1024-2048") + - protocol/port:source_ip (e.g., "tcp/22:103.104.105.106") + - protocol name only (e.g., "icmp", "tcp", "udp") + + Args: + port_entry (str): The port entry to validate + + Returns: + bool: True if valid, False otherwise + """ + import re + + # Protocol names only (e.g., "icmp", "tcp", "udp") + if port_entry.lower() in ["icmp", "tcp", "udp"]: + return True + + # For other formats, split by protocol and the rest + if "/" not in port_entry: + return False + + protocol, rest = port_entry.split("/", 1) + + # Validate protocol + if protocol.lower() not in ["tcp", "udp", "icmp"]: + return False + + # Check for source IP format (port:ip) + if ":" in rest: + port_part, ip_part = rest.split(":", 1) + + # Validate IP address format + ip_pattern = r"^(\d{1,3}\.){3}\d{1,3}$" + if not re.match(ip_pattern, ip_part): + return False + + # Also validate that IP octets are in valid range (0-255) + octets = ip_part.split(".") + if not all(0 <= int(octet) <= 255 for octet in octets): + return False + else: + port_part = rest + + # Validate port or port range + if "-" in port_part: + # Port range format + try: + start_port, end_port = port_part.split("-", 1) + start_port = int(start_port) + end_port = int(end_port) + + # Validate port range + if not (0 <= start_port <= 65535 and 0 <= end_port <= 65535 and start_port < end_port): + return False + except ValueError: + return False + else: + # Single port format + try: + port = int(port_part) + if not (0 <= port <= 65535): + return False + except ValueError: + return False + + return True + +def validate_payload(payload): + """ + Validate the input payload according to the specified rules and return a sanitized version. + + Args: + payload (dict): The input JSON object to validate. + + Returns: + dict: A sanitized version of the input payload containing only the expected parameters. + + Raises: + ValueError: If the payload fails any validation rule. + """ + import uuid # Added import for UUID validation + + # Ensure the input is a valid JSON object (dict) + if not isinstance(payload, dict): + raise ValueError("Input must be a valid JSON object (dict).") + + # Check for the top-level 'virtual_data_center' key and validate it as a UUID + if 'virtual_data_center' not in payload: + raise ValueError("Top-level 'virtual_data_center' key is missing.") + try: + uuid.UUID(payload['virtual_data_center'], version=4) + except ValueError: + raise ValueError("'virtual_data_center' must be a valid UUID.") + + # Check for the 'virtual-machines' key and ensure it is a list with at least one VM + if 'virtual-machines' not in payload: + raise ValueError("'virtual-machines' key is missing.") + if not isinstance(payload['virtual-machines'], list) or len(payload['virtual-machines']) == 0: + raise ValueError("'virtual-machines' must be a list with at least one VM.") + + # Initialize the sanitized payload + sanitized_payload = { + 'virtual_data_center': payload['virtual_data_center'], + 'virtual-machines': [] + } + + # Validate each VM in the 'virtual-machines' list and build the sanitized version + for vm in payload['virtual-machines']: + if not isinstance(vm, dict): + raise ValueError("Each virtual machine must be a dictionary.") + + # Check for required keys in the new format + if 'name' not in vm: + raise ValueError("Virtual machine is missing 'name'.") + if 'memory' not in vm: + raise ValueError("Virtual machine is missing 'memory'.") + if 'vcpu' not in vm: + raise ValueError("Virtual machine is missing 'vcpu'.") + + # Validate numeric fields + if not isinstance(vm['memory'], int) or vm['memory'] <= 0: + raise ValueError("'memory' must be a positive integer.") + if not isinstance(vm['vcpu'], int) or vm['vcpu'] <= 0: + raise ValueError("'vcpu' must be a positive integer.") + + # Initialize the sanitized VM + sanitized_vm = { + 'name': vm['name'], + 'memory': vm['memory'], + 'vcpu': vm['vcpu'] + } + + # Validate 'volumes' if present + if 'volumes' in vm: + if not isinstance(vm['volumes'], list): + raise ValueError("'volumes' must be a list.") + + sanitized_volumes = [] + for volume in vm['volumes']: + if not isinstance(volume, dict): + raise ValueError("Each volume must be a dictionary.") + + if 'name' not in volume or 'size_gb' not in volume: + raise ValueError("Volume is missing 'name' or 'size_gb'.") + + sanitized_volume = { + 'name': volume['name'], + 'size_gb': volume['size_gb'], + } + + + if not isinstance(volume['size_gb'], (int, float)) or volume['size_gb'] <= 0: + raise ValueError("'size_gb' must be a positive number.") + # Check if 'image' is present and validate it as a UUID + if 'image_id' in volume: + try: + uuid.UUID(volume['image_id']) + except ValueError: + raise ValueError(f"'{volume['image_id']}' is not a valid UUID.") + sanitized_volume['image_id'] = volume['image_id'] + + # Add optional volume fields if present + if 'boot' in volume: + if not isinstance(volume['boot'], bool): + raise ValueError("'boot' must be a boolean.") + sanitized_volume['boot'] = volume['boot'] + + if 'type' in volume: + sanitized_volume['type'] = volume['type'] + + if 'source' in volume: + sanitized_volume['source'] = volume['source'] + + + sanitized_volumes.append(sanitized_volume) + + sanitized_vm['volumes'] = sanitized_volumes + + # Validate 'networks' if present + if 'networks' in vm: + if not isinstance(vm['networks'], list): + raise ValueError("'networks' must be a list.") + + sanitized_networks = [] + for network in vm['networks']: + if not isinstance(network, dict): + raise ValueError("Each network must be a dictionary.") + + if 'id' not in network: + raise ValueError("Network is missing 'id'.") + + sanitized_network = { + 'id': network['id'] + } + + # Add optional network fields if present + if 'mac' in network: + # Could add MAC address format validation here + sanitized_network['mac'] = network['mac'] + + if 'ip_address' in network: + # Could add IP address format validation here + sanitized_network['ip_address'] = network['ip_address'] + + if 'open_ports' in network: + if not isinstance(network['open_ports'], list): + raise ValueError("'open_ports' must be a list.") + + # Validate each port entry + for port_entry in network['open_ports']: + if not isinstance(port_entry, str): + raise ValueError("Each port entry must be a string.") + + if not validate_port_entry(port_entry): + raise ValueError(f"Invalid port entry format: '{port_entry}'") + + sanitized_network['open_ports'] = network['open_ports'] + + sanitized_networks.append(sanitized_network) + + sanitized_vm['networks'] = sanitized_networks + + # Validate 'tags' if present + if 'tags' in vm: + if not isinstance(vm['tags'], dict): + raise ValueError("'tags' must be a dictionary.") + sanitized_vm['tags'] = vm['tags'] + + # Validate 'requirements' if present + if 'requirements' in vm: + if not isinstance(vm['requirements'], dict): + raise ValueError("'requirements' must be a dictionary.") + sanitized_vm['requirements'] = vm['requirements'] + + # Add the sanitized VM to the sanitized payload + sanitized_payload['virtual-machines'].append(sanitized_vm) + + return sanitized_payload + +@api_bp.route('/workloads/virtual_machines', methods=['POST']) +def add_VirtualMachine_workload(): + request_data = request.json + logger.debug(f"Request data: {request_data}") + + # Validate incoming data + validated_data = validate_payload(request_data) + + # TODO - Check if this user has permissions to VIEW and LAUNCH_VirtualMachine in this VDC + request_vdc = VirtualDataCenter.query.filter_by(id=uuid.UUID(validated_data['virtual_data_center'])).first() + + if request_vdc is None: + # Raise an error or return a response indicating that no VDC was found + abort(404, description="Virtual Data Center not found") # Flask's abort function + + all_workload_hosts = WorkloadHost.query.filter_by(deleted=0, region_id=request_vdc.region.id).all() + logger.debug(f"Found {len(all_workload_hosts)} hosts eligible for placement") + + if len(all_workload_hosts) == 0: + logger.warning(f"No hosts found in region {request_vdc.region.name} to place this request") + return f"No hosts found in region {request_vdc.region.name} to place this request", 400 + + client_response = [] + for VirtualMachine in validated_data['virtual-machines']: + # TODO - This is where we would slot in the 'placement' routine + random_host = random.choice(all_workload_hosts) + logger.info(f"Randomly selected host: {random_host}") + + # Create the Workload instance in the database + new_VirtualMachine = Workload( + name=VirtualMachine['name'], + workload_type="VirtualMachine", # Set workload_type to VirtualMachine + vdc_id=request_vdc.id, + status="pending-allocation", + launch_params=json.dumps(VirtualMachine ), + workload_host_id=random_host.id + ) + db.session.add(new_VirtualMachine) + db.session.commit() + logger.debug(f"VirtualMachine {new_VirtualMachine.id} workload added to DB") + + # Create volumes in the database + volumes = [] + for volume in VirtualMachine.get('volumes', []): + logger.debug(f"Volume details: {volume}") + _volume_source="" + _volume_image_id=None + + if "image_id" in volume: + logger.debug(f"Volume had image id specified {volume['image_id']}, confirming if it's a legitimate image ID or not") + + # Validate the image ID + try: + # Convert the image ID to a UUID object + image_id = uuid.UUID(volume['image_id']) + + # Query the image from the database + _image = Image.query.filter_by(id=image_id).first() + + if _image is None: + # If the image does not exist, return an error + return {"error": f"Image with ID {volume['image_id']} does not exist"}, 404 + logger.debug("Image seems legit") + # If the image exists, load the image ID into a variable + _volume_image_id = _image.id + _volume_source = _image.location + logger.debug(f"Image ID {image_id} is valid and loaded.") + + except ValueError: + # Handle the case where the image ID is not a valid UUID + return {"error": f"Invalid image ID format: {volume['image_id']}"}, 400 + else: + logger.debug(f"No image specified for this volume") + + logger.debug(f"Vol type is {volume['type']}") + _volume_path=volume['type'] + + new_volume = Volume( + name=volume['name'], + image_id=_volume_image_id, + type=_volume_path, + path=_volume_path, + size_gb=volume['size_gb'], + vdc_id=request_vdc.id, + source=_volume_source + ) + logger.debug(f"Volume spec: {new_volume.to_json()}") + db.session.add(new_volume) + db.session.commit() + volumes.append(new_volume) + + # Create network ports if networks are specified + networks = [] + for network in VirtualMachine.get('networks', []): + network_obj = Network.query.filter_by(id=uuid.UUID(network['id'])).first() + if network_obj: + port = network_obj.create_port(db.session, workload_id=new_VirtualMachine.id) + networks.append(port) + + # Send the request off to the websocket server to have this VirtualMachine launched + payload = { + "worker_id": random_host.id, + "task_type": "virtual-machine-create", + "job_details": { + "virtual_machine_name": VirtualMachine['name'], + "virtual_machine_id": new_VirtualMachine.id, + "desired_state": "running", + "virtual_machine_config": { + "memory": VirtualMachine['memory'], + "vcpu": VirtualMachine['vcpu'], + "volumes": [ + { + "id": vol.id, + "name": vol.name, + "size_gb": vol.size_gb, + "type": vol.type, + "path": vol.path, + **({"source": { # Include image object only if image_id exists + "type": "image", + "image_id": vol.image.id, + "path": vol.image.location, + "format": vol.image.format, + "checksum": vol.image.checksum + }} if vol.image_id else { + "source": "None" + }) # Conditionally add image object + } + for vol in volumes + ], + "networks": [ + { + "id": net.id, + "mac_address": net.mac_address + } + for net in networks + ] + } + } + } + logger.debug(f"Sending payload to websocket server {payload}") + headers = {"Content-Type": "application/json"} + + websocket_server_response = requests.post(websocket_server_url, data=json.dumps(payload), headers=headers) + websocket_server_response_data = websocket_server_response.json() + + if websocket_server_response.status_code == 201: + logger.info(f"Task created successfully for VirtualMachine {new_VirtualMachine.id}") + new_VirtualMachine.status = "pending-allocated" + db.session.add(new_VirtualMachine) + db.session.commit() + else: + logger.error(f"Creation request failed for VirtualMachine {new_VirtualMachine.id}") + new_VirtualMachine.status = "failed-allocation" + db.session.add(new_VirtualMachine) + db.session.commit() + + # Prepare the response to the API request + client_response.append({ + "virtual_machine_id": new_VirtualMachine.id, + "virtual_machine_name": new_VirtualMachine.name, + "VirtualMachine_status": new_VirtualMachine.status, + }) + + return client_response, 200 + + +@api_bp.route('/workloads/virtual_machines/status_update/', methods=['PUT']) +def edit_VirtualMachine_workload(virtual_machine_id): + data = request.json + + # Validate new_status + if data.get("new_status") not in ["running", "deleted", "stopped"]: + error_message = "Invalid status. Must be 'running' or 'deleted'." + logger.error(error_message) + return jsonify({"success": False, "message": error_message}), 400 + + # Validate system_vm_id and worker_id as UUIDs + try: + worker_id = uuid.UUID(data.get("worker_id")) + except (ValueError, TypeError) as e: + error_message = "Invalid UUID format for worker_id." + logger.error(f"{error_message} Error: {str(e)}") + return jsonify({"success": False, "message": error_message}), 400 + + # Validate virtual_machine_id as UUID + try: + virtual_machine_id = uuid.UUID(virtual_machine_id) + except (ValueError, TypeError) as e: + error_message = "Invalid UUID format for virtual_machine_id." + logger.error(f"{error_message} Error: {str(e)}") + return jsonify({"success": False, "message": error_message}), 400 + + # Validate timestamp as ISO format + try: + datetime.fromisoformat(data.get("timestamp")) + except (ValueError, TypeError) as e: + error_message = "Invalid ISO timestamp format." + logger.error(f"{error_message} Error: {str(e)}") + return jsonify({"success": False, "message": error_message}), 400 + + # Fetch the virtual machine from the database + try: + virtual_machine = Workload.query.filter_by(id=virtual_machine_id, workload_type="VirtualMachine").first_or_404() + except Exception as e: + error_message = f"Failed to fetch virtual machine with ID {virtual_machine_id}." + logger.error(f"{error_message} Error: {str(e)}") + return jsonify({"success": False, "message": error_message}), 404 + + # Check if the VM is actually on the reported worker + if virtual_machine.workload_host_id != worker_id: + error_message = f"Workload host in DB for VM ID {virtual_machine_id} is {virtual_machine.workload_host_id}, but was reported from {worker_id}. Ignoring." + logger.error(error_message) + return jsonify({"success": False, "message": error_message}), 400 + + # Update the status of the virtual machine + try: + new_status = data.get('new_status') + logger.info(f"Updating VirtualMachine ID: {virtual_machine_id} to new status {new_status}") + virtual_machine.status = new_status + + # If new status = 'started' + db.session.commit() + return jsonify({"success": True, "message": "Status updated successfully."}), 200 + except Exception as e: + error_message = f"Failed to update status for VirtualMachine ID: {virtual_machine_id}." + logger.error(f"{error_message} Error: {str(e)}") + db.session.rollback() + return jsonify({"success": False, "message": error_message}), 500 + + +@api_bp.route('/workloads/virtual_machines/', methods=['GET']) +def get_VirtualMachine_workload(workload_id): + try: + workload_uuid = uuid.UUID(workload_id) + except ValueError: + return jsonify({"error": "Invalid workload ID"}), 404 + + workload = Workload.query.filter( + Workload.id == workload_uuid, + Workload.workload_type == "VirtualMachine", + Workload.deleted == False + ).first_or_404() + logger.info(f"{workload.workload_type} {workload.deleted}") + return jsonify(workload.to_json()) + +@api_bp.route('/workloads/virtual_machines/', methods=['DELETE']) +def delete_VirtualMachine_workload(workload_id): + try: + workload_uuid = uuid.UUID(workload_id) + except ValueError: + return jsonify({"error": "Invalid workload ID"}), 404 + + _VirtualMachine = Workload.query.filter( + Workload.id == workload_uuid, + Workload.workload_type == "VirtualMachine", + Workload.deleted == False + ).first_or_404() + + payload = { + "worker_id": _VirtualMachine.workload_host_id, + "task_type": "virtual-machine-delete", + "job_details": { + "virtual_machine_id": _VirtualMachine.id, + "desired_state": "deleted" + } + } + logger.debug(f"Sending payload to websocket server {payload}") + headers = {"Content-Type": "application/json"} + + websocket_server_response = requests.post(websocket_server_url, data=json.dumps(payload), headers=headers) + websocket_server_response_data = websocket_server_response.json() + + if websocket_server_response.status_code == 201: + logger.info(f"Task created successfully for VirtualMachine {_VirtualMachine.id}") + _VirtualMachine.status = "pending-deleted" + db.session.add(_VirtualMachine) + db.session.commit() + else: + logger.error(f"Creation request failed for VirtualMachine {_VirtualMachine.id}") + _VirtualMachine.status = "failed-deleted" + db.session.add(_VirtualMachine) + db.session.commit() + + _VirtualMachine.soft_delete() + logger.info(f"Deleted VirtualMachine {_VirtualMachine.id}") + return jsonify({'message': 'VirtualMachine workload deleted successfully'}), 200 + +@api_bp.route('/workloads/virtual_machines', methods=['GET']) +def get_VirtualMachine_workloads(): + workloads = Workload.query.filter( + Workload.workload_type == "VirtualMachine", + Workload.deleted == False + ).all() + logger.debug(f"Found {len(workloads)} VirtualMachines") + return jsonify([workload.to_json() for workload in workloads]) diff --git a/app/controller/ui_routes.py b/app/controller/ui_routes.py index 566f273..dc7438a 100644 --- a/app/controller/ui_routes.py +++ b/app/controller/ui_routes.py @@ -1,7 +1,8 @@ from flask import request, jsonify, render_template, g, url_for -from app.models.models import Universe, Project, VirtualDataCenter, Workload, Network, Region +import requests +from app.models.models import Universe, Project, VirtualDataCenter, Workload, Region from app.controller import ui_bp - +from app.models.network import * # UI Routes with prefix @ui_bp.route('/') @@ -37,3 +38,27 @@ def networks(): def regions(): regions = Region.query.all() return render_template('regions.html', regions=regions) + + +@ui_bp.route("/vnc/") +def console(vm_id): + """Render the noVNC client page for a specific VM""" + # Fetch the WebSocket URL from the API + API_SERVER_URL = "http://localhost:5000" # API to get VNC session token + + # response = requests.post(f"{API_SERVER_URL}/get-console-url", json={"username": "test_user", "worker_id": vm_id}) + + # if response.status_code != 200: + # return "Error: Unable to get VNC session", 500 + + # console_data = response.json() + # console_url = console_data.get("console_url") + # Get VNC connection parameters from request args or use defaults + websocket_host = '127.0.0.1' + websocket_port = 6080 # Standard noVNC port + path = '' + + return render_template('console.html', + websocket_host=websocket_host, + websocket_port=websocket_port, + path=path) \ No newline at end of file diff --git a/app/models/models.py b/app/models/models.py index bf818a2..a58e433 100644 --- a/app/models/models.py +++ b/app/models/models.py @@ -133,34 +133,10 @@ class Workload(BaseModel): status = Column(String, nullable=True) container_id = Column(String, nullable=True) launch_params = Column(String, nullable=True) + time_to_provision = Column(Float, nullable=True) -class Network(BaseModel): - __tablename__ = "networks" - vdc_id = Column(UUID(as_uuid=True), ForeignKey("vdcs.id"), nullable=False) - ipv4_cidr = Column(String, nullable=True) - ipv4_gateway = Column(String, nullable=True) - ipv4_dns_servers = Column(String, nullable=True) - ipv6_cidr = Column(String, nullable=True) - ipv6_gateway = Column(String, nullable=True) - vni = Column(Integer, nullable=False) - encapsulation = Column(String, default="VXLAN", nullable=False) - ovs_bridge = Column(String, nullable=True) - - -class NetworkPort(BaseModel): - __tablename__ = "network_ports" - - # Foreign Keys - network_id = Column(UUID(as_uuid=True), ForeignKey("networks.id"), nullable=False) - workload_id = Column(UUID(as_uuid=True), ForeignKey("workloads.id"), nullable=True) - - # Network Port Properties - name = Column(String, nullable=True) - ip_address = Column(String, nullable=True) # IPv4 or IPv6 address - mac_address = Column(String, nullable=True) # MAC address - port_type = Column(String, nullable=True) # e.g., "public", "private", "management" - dns_servers = Column(String, nullable=True) # DNS servers (comma-separated) - subnet_mask = Column(String, nullable=True) # Subnet mask for the IP address + # Relationship to VolumeWorkloadMapping + volume_mappings = relationship("VolumeWorkloadMapping", back_populates="workload") class Image(BaseModel): __tablename__ = "images" @@ -169,13 +145,39 @@ class Image(BaseModel): size = Column(Float, nullable=False) os_family = Column(String, nullable=True) os_version = Column(String, nullable=True) + checksum = Column(String, nullable=True) + format = Column(String, nullable=True) + # Relationship to Volume + volumes = relationship("Volume", back_populates="image") class Volume(BaseModel): __tablename__ = "volumes" - volume_type = Column(Enum("Ceph", "Local", name="volume_type"), nullable=False) - path = Column(String, nullable=False) + path = Column(String, nullable=True) + type = Column(String, nullable=True) + source = Column(String, nullable=True) size_gb = Column(Float, nullable=False) + boot = Column(Boolean, nullable=False, default=False) vdc_id = Column(UUID(as_uuid=True), ForeignKey("vdcs.id"), nullable=False) + # Optional foreign key to link a volume to an image + image_id = Column(UUID(as_uuid=True), ForeignKey("images.id"), nullable=True) + + # Relationship to Image + image = relationship("Image", back_populates="volumes") + + # Relationship to VolumeWorkloadMapping + workload_mappings = relationship("VolumeWorkloadMapping", back_populates="volume") + +class VolumeWorkloadMapping(BaseModel): + __tablename__ = "volume_workload_mapping" + + # Composite primary key (volume_id and workload_id) + volume_id = Column(UUID(as_uuid=True), ForeignKey("volumes.id"), primary_key=True) + workload_id = Column(UUID(as_uuid=True), ForeignKey("workloads.id"), primary_key=True) + + # Relationships + volume = relationship("Volume", back_populates="workload_mappings") + workload = relationship("Workload", back_populates="volume_mappings") + class User(db.Model): __tablename__ = "users" diff --git a/app/models/network.py b/app/models/network.py new file mode 100644 index 0000000..f9c49a2 --- /dev/null +++ b/app/models/network.py @@ -0,0 +1,82 @@ +import uuid +import ipaddress +from sqlalchemy.orm import Session + +from datetime import datetime +from sqlalchemy import ( + Column, String, Boolean, DateTime, ForeignKey, Integer, Float, Text, Enum +) +from sqlalchemy.dialects.postgresql import UUID + +from .models import BaseModel + + +class NetworkPort(BaseModel): + __tablename__ = "network_ports" + + # Foreign Keys + network_id = Column(UUID(as_uuid=True), ForeignKey("networks.id"), nullable=False) + workload_id = Column(UUID(as_uuid=True), ForeignKey("workloads.id"), nullable=True) + + # Network Port Properties + name = Column(String, nullable=True) + ip_address = Column(String, nullable=True) # IPv4 or IPv6 address + mac_address = Column(String, nullable=True) # MAC address + port_type = Column(String, nullable=True) # e.g., "public", "private", "management" + dns_servers = Column(String, nullable=True) # DNS servers (comma-separated) + subnet_mask = Column(String, nullable=True) # Subnet mask for the IP address + + + +class Network(BaseModel): + __tablename__ = "networks" + vdc_id = Column(UUID(as_uuid=True), ForeignKey("vdcs.id"), nullable=False) + ipv4_cidr = Column(String, nullable=True) + ipv4_gateway = Column(String, nullable=True) + ipv4_dns_servers = Column(String, nullable=True) + ipv6_cidr = Column(String, nullable=True) + ipv6_gateway = Column(String, nullable=True) + vni = Column(Integer, nullable=False) + encapsulation = Column(String, default="VXLAN", nullable=False) + ovs_bridge = Column(String, nullable=True) + + def create_port(self, db: Session, workload_id: UUID = None, name: str = None, port_type: str = None): + # Define the prefix + prefix = "fa:16:4a" + + # Generate the MAC address with the specified prefix + mac_address = prefix + ':' + ':'.join(f'{byte:02x}' for byte in uuid.uuid4().bytes[-3:] ) + + # Generate a unique IP address within the network's CIDR range + ip_network = ipaddress.ip_network(self.ipv4_cidr) + used_ips = {port.ip_address for port in db.query(NetworkPort).filter(NetworkPort.network_id == self.id).all()} + + for ip in ip_network.hosts(): + if str(ip) not in used_ips: + ip_address = str(ip) + break + else: + raise ValueError("No available IP addresses in the network") + + # Check for MAC address conflicts + if db.query(NetworkPort).filter(NetworkPort.mac_address == mac_address).first(): + raise ValueError("MAC address conflict detected") + + # Create the new port + new_port = NetworkPort( + network_id=self.id, + workload_id=workload_id, + name=name, + ip_address=ip_address, + mac_address=mac_address, + port_type=port_type, + dns_servers=self.ipv4_dns_servers, + subnet_mask=str(ip_network.netmask), + ) + + db.add(new_port) + db.commit() + db.refresh(new_port) + + return new_port + diff --git a/app/static/novnc b/app/static/novnc new file mode 160000 index 0000000..b25675e --- /dev/null +++ b/app/static/novnc @@ -0,0 +1 @@ +Subproject commit b25675e05290817024fa5ba89581d9cdc12f8992 diff --git a/app/static/style.css b/app/static/style.css index 3f4d61a..553ffba 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -4,3 +4,60 @@ body { h2 { margin-bottom: 20px; } +body { + margin: 0; + padding: 0; + height: 100vh; + display: flex; + flex-direction: column; + font-family: Arial, sans-serif; +} + +#novnc-container { + flex-grow: 1; + background-color: #f0f0f0; + position: relative; +} + +#vnc-canvas { + width: 100%; + height: 100%; +} + +#connection-status { + position: absolute; + top: 10px; + left: 10px; + padding: 5px 10px; + background-color: rgba(0, 0, 0, 0.7); + color: white; + border-radius: 5px; + z-index: 100; +} + +.controls { + padding: 10px; + background-color: #eaeaea; + border-bottom: 1px solid #ccc; + display: flex; + justify-content: space-between; +} + +button { + padding: 6px 12px; + margin: 0 4px; + background-color: #4CAF50; + color: white; + border: none; + border-radius: 4px; + cursor: pointer; +} + +button:hover { + background-color: #45a049; +} + +button:disabled { + background-color: #cccccc; + cursor: not-allowed; +} \ No newline at end of file diff --git a/app/templates/console.html b/app/templates/console.html new file mode 100644 index 0000000..a52f2cb --- /dev/null +++ b/app/templates/console.html @@ -0,0 +1,416 @@ + + + + + noVNC + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
noVNC encountered an error:
+
+
+
+
+ + +
+ +
+
+ +
+ +

no
VNC

+ +
+ + + + + +
+ +
+ + + +
+
+ + + + + + +
+
+ + + +
+
+
+ Power +
+ + + +
+
+ + + +
+
+
+ Clipboard +
+

+ Edit clipboard content in the textarea below. +

+ +
+
+ + + + + + +
+
+
+ Settings +
+
    +
  • + +
    • +
  • + +
    • +
    • +
  • + +
    • +
  • + + +
    • +
    • +
  • +
    Advanced
    +
      +
    • + + +
      • +
    • + + +
      • +
      • +
    • + + +
      • +
    • +
      WebSocket
      +
        +
      • + +
        • +
      • + + +
        • +
      • + + +
        • +
      • + + +
        • +
      +
      • +
      • +
    • + +
      • +
    • + + +
      • +
      • +
    • + +
      • +
      • + +
    • + +
      • +
    +
    • +
    • +
  • + Version: + +
    • +
+
+
+ + + + +
+
+ +
+ +
+
+
+
+ + +
+ + +
+
+ +
+ +
+
+
+ + +
+
+
+ Server identity +
+
+ The server has provided the following identifying information: +
+
+ Fingerprint: + +
+
+ Please verify that the information is correct and press + "Approve". Otherwise press "Reject". +
+
+ + +
+
+
+ + +
+
+
+ Credentials +
+
+ + +
+
+ + +
+
+ +
+
+
+ + +
+
+
+ +
+ +
+ + +
+ + +
+ + + + \ No newline at end of file diff --git a/bootstrap_db.py b/bootstrap_db.py index f034857..2db2b18 100644 --- a/bootstrap_db.py +++ b/bootstrap_db.py @@ -1,50 +1,18 @@ import uuid from datetime import datetime +import logging from sqlalchemy.orm import sessionmaker +from sqlalchemy.exc import SQLAlchemyError from app import db, app from app.models.models import ( User, Universe, Project, Region, VirtualDataCenter, WorkloadHost, - WorkloadHostFixedResource, WorkloadHostPooledResource, Workload, Network, Image, Volume, Label, RegionAccess, NetworkPort + WorkloadHostFixedResource, WorkloadHostPooledResource, Workload, Image, Volume, Label, RegionAccess ) +from app.models.network import * -# Helper function to generate IP addresses within a subnet -def generate_ip_addresses(subnet, count): - """ - Generates a list of IP addresses within a given subnet. - - Args: - subnet (str): The subnet in CIDR notation (e.g., "192.168.1.0/24"). - count (int): The number of IP addresses to generate. - - Returns: - list: A list of IP addresses. - """ - base_ip = subnet.split('/')[0] # Extract the base IP (e.g., "192.168.1.0") - ip_parts = list(map(int, base_ip.split('.'))) # Split into octets - ip_addresses = [] - - for i in range(1, count + 1): # Start from 1 to avoid the network address - new_ip = f"{ip_parts[0]}.{ip_parts[1]}.{ip_parts[2]}.{ip_parts[3] + i}" - ip_addresses.append(new_ip) - - return ip_addresses - -# Helper function to generate MAC addresses -def generate_mac_addresses(count): - """ - Generates a list of unique MAC addresses. - - Args: - count (int): The number of MAC addresses to generate. - - Returns: - list: A list of MAC addresses. - """ - mac_addresses = [] - for i in range(count): - mac = f"00:1A:2B:3C:4D:{i:02X}" # Generate a unique MAC address - mac_addresses.append(mac) - return mac_addresses +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) # Initialize session with application context with app.app_context(): @@ -53,137 +21,273 @@ with app.app_context(): db.create_all() def create_users(): - users = [ - User( - id=uuid.uuid4(), - first_name=f"User {i}", - last_name="Doe", - email=f"user{i}@example.com", - oidc_id=str(uuid.uuid4()) - ) - for i in range(10) # Create 10 users - ] - session.add_all(users) - session.commit() - return users + try: + users = [ + User( + id=uuid.uuid4(), + first_name=f"User {i}", + last_name="Doe", + email=f"user{i}@example.com", + oidc_id=str(uuid.uuid4()) + ) + for i in range(10) # Create 10 users + ] + session.add_all(users) + session.commit() + logger.info(f"Created {len(users)} users.") + return users + except SQLAlchemyError as e: + session.rollback() + logger.error(f"Error creating users: {e}") + raise def create_universes(users): - universes = [ - Universe( - id=uuid.uuid4(), - name=f"Universe {i}", - description=f"A test universe {i}", - created_by=users[i % len(users)].id # Assign created_by to a user - ) - for i in range(10) # Create 10 universes - ] - session.add_all(universes) - session.commit() - return universes + try: + universes = [ + Universe( + id=uuid.uuid4(), + name=f"Universe {i}", + description=f"A test universe {i}", + created_by=users[i % len(users)].id # Assign created_by to a user + ) + for i in range(10) # Create 10 universes + ] + session.add_all(universes) + session.commit() + logger.info(f"Created {len(universes)} universes.") + return universes + except SQLAlchemyError as e: + session.rollback() + logger.error(f"Error creating universes: {e}") + raise def create_projects(universes): - projects = [ - Project( - id=uuid.uuid4(), - name=f"Project {i}", - universe_id=universes[i % len(universes)].id, # Assign to a universe - created_by=universes[i % len(universes)].created_by # Assign created_by to the universe's creator - ) - for i in range(10) # Create 10 projects - ] - session.add_all(projects) - session.commit() - return projects + try: + projects = [ + Project( + id=uuid.uuid4(), + name=f"Project {i}", + universe_id=universes[i % len(universes)].id, # Assign to a universe + created_by=universes[i % len(universes)].created_by # Assign created_by to the universe's creator + ) + for i in range(10) # Create 10 projects + ] + session.add_all(projects) + session.commit() + logger.info(f"Created {len(projects)} projects.") + return projects + except SQLAlchemyError as e: + session.rollback() + logger.error(f"Error creating projects: {e}") + raise def create_regions(users): - regions = [ - Region( - id=uuid.uuid4(), - name=f"Region {i}", - country=f"Country {i}", - abbreviation=f"REG{i}", - created_by=users[i % len(users)].id # Assign created_by to a user - ) - for i in range(10) # Create 10 regions - ] - session.add_all(regions) - session.commit() - return regions + try: + regions = [ + Region( + id=uuid.uuid4(), + name=f"Region {i}", + country=f"Country {i}", + abbreviation=f"REG{i}", + created_by=users[i % len(users)].id # Assign created_by to a user + ) + for i in range(10) # Create 10 regions + ] + session.add_all(regions) + session.commit() + logger.info(f"Created {len(regions)} regions.") + return regions + except SQLAlchemyError as e: + session.rollback() + logger.error(f"Error creating regions: {e}") + raise def create_vdcs(projects, regions): - vdcs = [ - VirtualDataCenter( - id=uuid.uuid4(), - name=f"VDC {i}", - project_id=projects[i % len(projects)].id, # Assign to a project - region_id=regions[i % len(regions)].id, # Assign to a region - created_by=projects[i % len(projects)].created_by # Assign created_by to the project's creator - ) - for i in range(10) # Create 10 VDCs - ] - session.add_all(vdcs) - session.commit() - return vdcs + try: + vdcs = [ + VirtualDataCenter( + id=uuid.uuid4(), + name=f"VDC {i}", + project_id=projects[i % len(projects)].id, # Assign to a project + region_id=regions[i % len(regions)].id, # Assign to a region + created_by=projects[i % len(projects)].created_by # Assign created_by to the project's creator + ) + for i in range(10) # Create 10 VDCs + ] + session.add_all(vdcs) + session.commit() + logger.info(f"Created {len(vdcs)} VDCs.") + return vdcs + except SQLAlchemyError as e: + session.rollback() + logger.error(f"Error creating VDCs: {e}") + raise def create_networks(vdcs): - networks = [ - Network( - name=f"Network {i}", - vdc_id=vdcs[i % len(vdcs)].id, # Assign to a VDC - vni=i + 1, # Unique VNI for each network - ipv4_cidr=f"192.168.{i}.0/24", # Assign a unique IPv4 subnet - ipv4_gateway=f"192.168.{i}.1" # Assign a gateway IP - ) - for i in range(10) # Create 10 networks - ] - session.add_all(networks) - session.commit() - return networks - - def create_network_ports(networks): - for network in networks: - # Generate IP and MAC addresses for ports - ip_addresses = generate_ip_addresses(network.ipv4_cidr, 3) # Create 3 ports per network - mac_addresses = generate_mac_addresses(3) # Generate 3 unique MAC addresses - - # Create network ports - ports = [ - NetworkPort( - network_id=network.id, - ip_address=ip_addresses[i], - mac_address=mac_addresses[i], - port_type="private", # Default port type - status="active" # Default state + try: + networks = [ + Network( + name=f"Network {i}", + vdc_id=vdcs[i % len(vdcs)].id, # Assign to a VDC + vni=i + 1, # Unique VNI for each network + ipv4_cidr=f"192.168.{i}.0/24", # Assign a unique IPv4 subnet + ipv4_gateway=f"192.168.{i}.1" # Assign a gateway IP ) - for i in range(3) # Create 3 ports per network + for i in range(10) # Create 10 networks ] - session.add_all(ports) - session.commit() - return ports + session.add_all(networks) + session.commit() + logger.info(f"Created {len(networks)} networks.") + return networks + except SQLAlchemyError as e: + session.rollback() + logger.error(f"Error creating networks: {e}") + raise + + def create_network_ports(networks: dict[Network]): + try: + ports = [] + for network in networks: + for i in range(3): + ports.append(network.create_port(session)) + session.commit() + logger.info(f"Created {len(ports)} network ports.") + return ports + except SQLAlchemyError as e: + session.rollback() + logger.error(f"Error creating network ports: {e}") + raise def create_region_access(projects, regions): - region_access_entries = [ - RegionAccess( - project_id=projects[i % len(projects)].id, # Assign to a project - region_id=regions[i % len(regions)].id # Assign to a region - ) - for i in range(10) # Create 10 RegionAccess entries - ] - session.add_all(region_access_entries) - session.commit() - return region_access_entries + try: + region_access_entries = [ + RegionAccess( + project_id=projects[i % len(projects)].id, # Assign to a project + region_id=regions[i % len(regions)].id # Assign to a region + ) + for i in range(10) # Create 10 RegionAccess entries + ] + session.add_all(region_access_entries) + session.commit() + logger.info(f"Created {len(region_access_entries)} region access entries.") + return region_access_entries + except SQLAlchemyError as e: + session.rollback() + logger.error(f"Error creating region access entries: {e}") + raise - def bootstrap_database(): - print("Bootstrapping database...") - users = create_users() - universes = create_universes(users) - projects = create_projects(universes) - regions = create_regions(users) - vdcs = create_vdcs(projects, regions) - networks = create_networks(vdcs) - network_ports = create_network_ports(networks) - region_access_entries = create_region_access(projects, regions) - print("Database bootstrapped successfully!") + def create_images(): + try: + images = [] + for i in range(5): # Create 5 sample images + # Randomly decide if the image location is a URL or a local path + is_http = i % 2 == 0 # Alternate between HTTP and local paths + location = ( + f"http://example.com/images/image_{i}.img" if is_http + else f"/images/image_{i}.img" + ) + location_type = "HTTP" if is_http else "Local" + + # Randomly assign image format (qcow2 or raw) + image_format = "qcow2" if i % 2 == 0 else "raw" + + images.append( + Image( + id=uuid.uuid4(), + name=f"Image {i}", + description=f"A sample image {i}", + location=location, + location_type=location_type, + size=10.0 * (i + 1), # Size in GB + os_family="Linux", + os_version=f"20.0{i}", + checksum=f"checksum_{i}", + format=image_format, # Add image format + created_by=uuid.uuid4() # Assign a random created_by user ID + ) + ) + + session.add_all(images) + session.commit() + logger.info(f"Created {len(images)} images.") + return images + except SQLAlchemyError as e: + session.rollback() + logger.error(f"Error creating images: {e}") + raise + + +def bootstrap_database(): + try: + logger.info("Bootstrapping database...") + + # Check and create users if none exist + if session.query(User).count() == 0: + users = create_users() + else: + users = session.query(User).all() + logger.info("Users already exist, skipping creation.") + + # Check and create universes if none exist + if session.query(Universe).count() == 0: + universes = create_universes(users) + else: + universes = session.query(Universe).all() + logger.info("Universes already exist, skipping creation.") + + # Check and create projects if none exist + if session.query(Project).count() == 0: + projects = create_projects(universes) + else: + projects = session.query(Project).all() + logger.info("Projects already exist, skipping creation.") + + # Check and create regions if none exist + if session.query(Region).count() == 0: + regions = create_regions(users) + else: + regions = session.query(Region).all() + logger.info("Regions already exist, skipping creation.") + + # Check and create VDCs if none exist + if session.query(VirtualDataCenter).count() == 0: + vdcs = create_vdcs(projects, regions) + else: + vdcs = session.query(VirtualDataCenter).all() + logger.info("VDCs already exist, skipping creation.") + + # Check and create networks if none exist + if session.query(Network).count() == 0: + networks = create_networks(vdcs) + else: + networks = session.query(Network).all() + logger.info("Networks already exist, skipping creation.") + + # Check and create images if none exist + if session.query(Image).count() == 0: + images = create_images() + else: + images = session.query(Image).all() + logger.info("Images already exist, skipping creation.") + + # Check and create network ports if none exist + if session.query(NetworkPort).count() == 0: + network_ports = create_network_ports(networks) + else: + network_ports = session.query(NetworkPort).all() + logger.info("Network ports already exist, skipping creation.") + + # Check and create region access entries if none exist + if session.query(RegionAccess).count() == 0: + region_access_entries = create_region_access(projects, regions) + else: + region_access_entries = session.query(RegionAccess).all() + logger.info("Region access entries already exist, skipping creation.") + + logger.info("Database bootstrapping completed successfully!") + except Exception as e: + logger.error(f"Error bootstrapping database: {e}") + raise if __name__ == "__main__": bootstrap_database() \ No newline at end of file diff --git a/goals.md b/goals.md index aea1214..b1a95d3 100644 --- a/goals.md +++ b/goals.md @@ -50,3 +50,163 @@ Spawn the VM Network port is created in OVS with known MAC VM is spawned with known name and ID Response sent to server with status of VM + + +VM launch flow +User requests vm launch +We create and add networkPort and virtual machine objects to DB +Craft the 'create-vm' task to send to the worker accordingly + +--- +VM Launch payload +{ + "vdc": "f9640ab7-3e8f-430a-83a7-abd0a2102fa3", + "virtual-machines": [ + { + "name": "test_vm_name", + "memory": 2048, + "vcpu": 2, + "volumes": [ + { + "name": "disk1", + "size_gb": 20, + "boot": true, + "type": "local", + "image":"image id or 'blank'" + }, + { + "name": "disk2", + "size_gb": 10, + "type": "Ceph" + } + ], + "networks": [ + { + "id": "{networkID}", + "mac":"00:11:22:33:44:55", + "ip_address":"192.168.50.124", + "open_ports":["tcp/80","udp/81","icmp","tcp/1024-2048","tcp/22:103.104.105.106"] + } + ], + "tags": { + "tag1": "value", + "tag3": "value" + } + , + "requirements": { + "tag": "value" + } + + } + ] +} +----------- +Payload sent to worker from API server +----------- +{ + "worker_id":"UUID(""d266b29d-a888-430d-8873-151c41fd34e3"")", + "task_type":"virtual-machine-create", + "job_details":{ + "virtual_machine_name":"my-vm", + "virtual_machine_id":"UUID(""16cb0f4a-d445-48c2-ab53-b8c9821e8c68"")", + "desired_state":"running", + "virtual_machine_config":{ + "memory":2048, + "vcpu":2, + "volumes":[ + { + "id":"UUID(""e6553965-6039-44e0-96cb-e0094f3b60be"")", + "name":"v1", + "size_gb":11.0, + "type":"None", + "source":"vol.source" + }, + { + "id":"UUID(""7309cd8e-9845-4b16-bae2-092dcb78fbea"")", + "name":"v2", + "size_gb":12.0, + "type":"None", + "source":"vol.source" + }, + { + "id":"UUID(""79cafd2e-f29e-4a70-b6f1-2d8ae6d07d05"")", + "name":"v3", + "size_gb":13.0, + "type":"None", + "source":"vol.source" + } + ], + "networks":[ + + ] + } + } +} + +Image \ Volume flow + +Create a blank volume locally +Input: + type: local + source: None +Output: + type: local + source: None + path: None + volume_id: {volumeID} + +Create a blank volume on ceph-highspeed +Input: + type: ceph-highspeed + source: None +Output + type: ceph + path: highspeed + source: None + volume_id: {volumeID} + +Create a volume locally from a URL image +Input + type: local + source: {imageID} +Output: + type: local + path: None + source: + type: image + url: http://some-url/image.qcow2?secret=dd + checksum: CHECKSUM + format: qcow2\raw + volume_id: {volumeID} + +Create a volume on ceph-nvme from a URL image +Input + type: ceph-nvme + source: {imageID} +Output: + type: ceph + path: nvme + source: + type: image + url: http://some-url/image.qcow2?secret=dd + checksum: CHECKSUM + format: qcow2\raw + volume_id: {volumeID} + + + +Create a volume locally from a ceph-slow image +Input + type: local + source: {imageID} +Output: + type: local + path: None + source: + type: image + url: rbd://volumes-sloq/image7 + checksum: CHECKSUM + format: raw + volume_id: {volumeID} + +Create a volume on ceph from a ceph-nvme image diff --git a/libvirt_test.py.py b/libvirt_test.py.py new file mode 100644 index 0000000..9ada488 --- /dev/null +++ b/libvirt_test.py.py @@ -0,0 +1,132 @@ +import logging +import os +import time +import libvirt +import uuid +# from libvirt_vm_task import LibvirtVMTask # Import the LibvirtVMTask class +from worker_tasks.libvirt import LibvirtVMTask + +def test_vm_with_volume_management(): + """ + Test function to demonstrate VM creation with automatic volume management + and proper cleanup on deletion. + """ + # Setup logging + logger = logging.getLogger("vm_test") + logger.setLevel(logging.INFO) + handler = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + # Create a unique VM name with timestamp + test_vm_name = f"test-vm-{int(time.time())}" + vm_id=str(uuid.uuid4()) + logger.info(f"Starting test for VM: {vm_id}") + + try: + # Step 1: Define VM configuration with volume that doesn't exist yet + # The script will automatically create it + vm_config = { + "vm_name": test_vm_name, + "vm_id": vm_id, + "desired_state": "running", + "vm_config": { + "memory": 2048, + "vcpu": 2, + "volumes": [ + {"id": "disk1", "size_gb": 20, "format": "qcow2"}, + {"id": "disk2", "size_gb": 10, "format": "qcow2"} + ], + "networks": [ + {"name": "default", "mac_address": "52:54:00:12:34:56"} + ], + } + } + + + + config = { + "default_volume_path": "/tmp" + } + # Step 2: Create and start VM - this will also create the volume + logger.info("Creating and starting VM with automatic volume creation") + task = LibvirtVMTask(vm_config, config, logger) + result = task.execute() + + if result["success"]: + logger.info(f"VM created successfully: {result['response']['status']}") + + # Step 3: Verify VM is running + conn = libvirt.open("qemu:///system") + try: + domain = conn.lookupByName(vm_id) + if domain.isActive(): + logger.info("Verified: VM is running") + else: + logger.error("VM exists but is not running") + except libvirt.libvirtError as e: + logger.error(f"Failed to find VM: {e}") + finally: + conn.close() + + + + # Step 5: Wait a moment before deleting + logger.info("Waiting 5 seconds before deletion...") + time.sleep(5) + + # Step 6: Delete VM - this will also delete the volume + logger.info("Deleting VM and associated volumes") + delete_config = { + "vm_id": vm_id, + "desired_state": "deleted" + } + delete_task = LibvirtVMTask(delete_config, config, logger) + delete_result = delete_task.execute() + + if delete_result["success"]: + logger.info(f"VM deleted successfully: {delete_result['response']['status']}") + + # Step 7: Verify VM was deleted + conn = libvirt.open("qemu:///system") + try: + conn.lookupByName(vm_id) + logger.error("VM still exists after deletion!") + except libvirt.libvirtError: + logger.info("Verified: VM was successfully deleted") + finally: + conn.close() + + + else: + logger.error(f"Failed to delete VM: {delete_result}") + else: + logger.error(f"Failed to create VM: {result}") + + except Exception as e: + logger.error(f"Test failed with error: {e}") + + # Emergency cleanup in case of test failure + try: + # Try to delete VM if it exists + conn = libvirt.open("qemu:///system") + try: + domain = conn.lookupByName(vm_id) + if domain.isActive(): + domain.destroy() + domain.undefine() + logger.info("Emergency cleanup: VM deleted") + except libvirt.libvirtError: + # VM doesn't exist, no cleanup needed + pass + finally: + conn.close() + except Exception as cleanup_error: + logger.error(f"Emergency cleanup failed: {cleanup_error}") + + + logger.info("Test completed") + +if __name__ == "__main__": + test_vm_with_volume_management() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c71587c..cb60560 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,10 +9,4 @@ pymysql flask_sqlalchemy flask_migrate websocket-client -streamlit -aiohttp -asyncio -docker -tabulate -psutil -python-dotenv \ No newline at end of file +streamlit \ No newline at end of file diff --git a/start_containers..sh b/start_containers.sh similarity index 100% rename from start_containers..sh rename to start_containers.sh diff --git a/streamlit_server.py b/streamlit_server.py index b407725..c23f316 100644 --- a/streamlit_server.py +++ b/streamlit_server.py @@ -1,6 +1,6 @@ import streamlit as st from streamlit_server.views import sidebar, universes, projects, vdcs, networks -from streamlit_server.views import details, regions, hosts, images, containers, websocket +from streamlit_server.views import details, regions, hosts, images, containers, websocket, virtual_machines from streamlit_server.utils.session import init_session_state def main(): @@ -12,11 +12,6 @@ def main(): init_session_state() - # Print the values in the Streamlit app - print("PRE Selected Resource:", st.session_state.selected_resource) - print("PRE View Type:", st.session_state.view_type) - print("PRE Selected Resource Type:", st.session_state.selected_resource_type) - print("PRE Current View:", st.session_state.current_view) sidebar.render() @@ -24,11 +19,7 @@ def main(): st.warning("Please enter your API key in the sidebar to continue.") return - # Print the values in the Streamlit app - print("POST Selected Resource:", st.session_state.selected_resource) - print("POST View Type:", st.session_state.view_type) - print("POST Selected Resource Type:", st.session_state.selected_resource_type) - print("POST Current View:", st.session_state.current_view) + if st.session_state.view_type == 'list': if st.session_state.current_view == "Universes": @@ -55,6 +46,9 @@ def main(): elif st.session_state.current_view == "Containers": print("Rendering: Containers") containers.render() + elif st.session_state.current_view == "Virtual Machines": + print("Rendering: Virtual Machines") + virtual_machines.render() elif st.session_state.current_view == "Websocket": print("Rendering: Websocket") websocket.render() diff --git a/streamlit_server/views/containers.py b/streamlit_server/views/containers.py index 0533824..a20b941 100644 --- a/streamlit_server/views/containers.py +++ b/streamlit_server/views/containers.py @@ -2,19 +2,31 @@ import json import streamlit as st from streamlit_server.utils.helpers import format_timestamp -def render(): + +def render_list(): """ Renders a list of all container workloads. """ st.header("Container Workload Management") - + + # Initialize session state variables for delete all confirmation + if 'container_delete_all_confirmation_shown' not in st.session_state: + st.session_state.container_delete_all_confirmation_shown = False + if 'container_delete_all_confirmed' not in st.session_state: + st.session_state.container_delete_all_confirmed = False + # Initialize session state variables for delete confirmation + if 'container_delete_confirmation_shown' not in st.session_state: + st.session_state.container_delete_confirmation_shown = False + if 'container_delete_confirmed' not in st.session_state: + st.session_state.container_delete_confirmed = False + # Fetch all VDCs for the dropdown - vdcs = st.session_state.client.get_vdcs() + vdcs = st.session_state.client.get_virtual_data_centers() vdc_options = {vdc['name']: vdc['id'] for vdc in vdcs} if vdcs else {} - + # Add a dropdown to select VDC selected_vdc_name = st.selectbox("Filter by Virtual Data Center", list(vdc_options.keys())) - + # Fetch all Workloads_hosts for the dropdown workload_hosts = st.session_state.client.get_workload_hosts() workload_host_options = {workload_host['hostname']: workload_host['id'] for workload_host in workload_hosts} if workload_hosts else {} @@ -42,11 +54,11 @@ def render(): }) if result: st.success("Container workload created successfully!") - + # List Container Workloads st.subheader("Existing Container Workloads") containers = st.session_state.client.get_containers() - + if containers: # Filter containers based on the selected VDC if selected_vdc_name: @@ -55,6 +67,40 @@ def render(): else: filtered_containers = containers + # Delete All functionality + if not st.session_state.container_delete_all_confirmation_shown: + if st.button("Delete All Containers", key="delete_all_containers", type="primary", help="This will delete all visible containers"): + st.session_state.container_delete_all_confirmation_shown = True + st.rerun() + + # Show confirmation if delete was clicked + if st.session_state.container_delete_all_confirmation_shown and not st.session_state.container_delete_all_confirmed: + st.warning("Are you sure you want to delete all containers? This action cannot be undone.") + confirm_col1, confirm_col2 = st.columns([1, 1]) + with confirm_col1: + if st.button("Yes, Delete All", key="confirm_delete_all_containers"): + st.session_state.container_delete_all_confirmed = True + st.rerun() + with confirm_col2: + if st.button("Cancel", key="cancel_delete_all_containers"): + st.session_state.container_delete_all_confirmation_shown = False + st.rerun() + + # Perform deletion if confirmed + if st.session_state.container_delete_all_confirmed: + deletion_count = 0 + for container in filtered_containers: + # Call the delete API for each container + result = st.session_state.client.delete_container(container['id']) + if result: + deletion_count += 1 + + st.success(f"Successfully deleted {deletion_count} containers.") + # Reset the confirmation flags + st.session_state.container_delete_all_confirmation_shown = False + st.session_state.container_delete_all_confirmed = False + + # Display the containers for container in filtered_containers: with st.container(): col1, col2, col3 = st.columns([3, 1, 1]) @@ -78,14 +124,110 @@ def render(): st.text(f"Image: {container_name}") with col2: st.text(f"ID: {container['id']}") - with col3: + with col3: if st.button("View Details", key=f"view_{container['id']}"): - print("Button is pressed") st.session_state.selected_resource = container st.session_state.view_type = 'detail' st.session_state.selected_resource_type = 'container' st.rerun() + # st.switch_page("streamlit_server/views/containers.py&container_id=d", query_params={"team": "streamlit"}) + else: st.info("No container workloads found.") + +def render_detail(): + + """ + Renders the details of a specific container workload. + """ + + container = st.session_state.client.get_container(st.session_state.selected_resource['id']) + if not container: + st.error("Container workload not found.") + st.session_state.selected_resource = None + st.session_state.view_type = 'list' + st.session_state.selected_resource_type = None + return + + st.title(f"Container Workload: {container['container_id']}") + if st.button("Back to list view"): + st.session_state.view_type = 'list' + st.rerun() + + st.header("Container Workload Details") + col1, col2 = st.columns(2) + with col1: + st.markdown(f"**ID:** `{container['id']}`") + st.markdown(f"**Created:** {format_timestamp(container.get('created_at', 'Unknown'))}") + st.markdown(f"**VDC ID:** {container['vdc_id']}") + st.markdown(f"**Workload Host ID:** {container.get('container_host_id', 'N/A')}") + with col2: + if container.get('updated_at'): + st.markdown(f"**Last Updated:** {format_timestamp(container['updated_at'])}") + st.markdown(f"**Status:** {container['status']}") + st.markdown(f"**Launch Parameters:** {container.get('launch_params', 'N/A')}") + + # Edit Container Workload Form + with st.expander("Edit Container Workload"): + with st.form("edit_container"): + new_status = st.selectbox("Status", ["running", "stopped", "pending","failed-deleted","pending-deleted","pending-allocation","failed-allocation","pending-allocated"], index=["running", "stopped", "pending","failed-deleted","pending-deleted","pending-allocation","failed-allocation","pending-allocated"].index(container['status'])) + new_launch_params = st.text_input("Launch Parameters", value=container.get('launch_params', '')) + submit = st.form_submit_button("Update Container Workload") + + if submit: + result = st.session_state.client.edit_container(container['id'], { + "status": new_status, + "launch_params": new_launch_params + }) + if result: + st.success("Container workload updated successfully!") + st.session_state.selected_resource = container + st.session_state.view_type = 'detail' + st.session_state.selected_resource_type = 'container' + st.rerun() + + + + # Delete functionality + if not st.session_state.container_delete_confirmation_shown: + if st.button("Delete container", key="delete_container", type="primary", help="This will delete visible container"): + st.session_state.container_delete_confirmation_shown = True + st.rerun() + + # Show confirmation if delete was clicked + if st.session_state.container_delete_confirmation_shown and not st.session_state.container_delete_confirmed: + st.warning("Are you sure you want to delete container? This action cannot be undone.") + confirm_col1, confirm_col2 = st.columns([1, 1]) + with confirm_col1: + if st.button("Yes, Delete", key="confirm_delete_container"): + st.session_state.container_delete_confirmed = True + st.rerun() + with confirm_col2: + if st.button("Cancel", key="cancel_delete_container"): + st.session_state.container_delete_confirmation_shown = False + st.rerun() + + # Perform deletion if confirmed + if st.session_state.container_delete_confirmed: + result = st.session_state.client.delete_container(container['id']) + st.session_state.container_delete_confirmation_shown = False + st.session_state.container_delete_confirmed = False + if result: + st.success("Container workload deleted successfully!") + st.session_state.selected_resource = None + st.session_state.view_type = 'list' + st.session_state.selected_resource_type = 'container' + st.rerun() + + st.success(f"Successfully deleted container.") + # Reset the confirmation flags + st.session_state.container_delete_confirmation_shown = False + st.session_state.container_delete_confirmed = False + + +if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='container': + render_detail() +else: + render_list() \ No newline at end of file diff --git a/streamlit_server/views/dashboard.py b/streamlit_server/views/dashboard.py new file mode 100644 index 0000000..5bcc773 --- /dev/null +++ b/streamlit_server/views/dashboard.py @@ -0,0 +1,34 @@ +import streamlit as st +import pandas as pd +from streamlit_server.utils.helpers import format_timestamp + +# def render(): +st.header("Dashboard") + +workload_hosts = st.session_state.client.get_workload_hosts() + +st.subheader("Workload Hosts") +if workload_hosts: + workload_hosts_df = pd.DataFrame(workload_hosts) + st.dataframe(workload_hosts_df) +else: + st.info("No workload hosts found.") + +images = st.session_state.client.get_images() + +st.subheader("Images") +if images: + images_df = pd.DataFrame(images) + st.dataframe(images_df) +else: + st.info("No workload hosts found.") + + +regions = st.session_state.client.get_regions() + +st.subheader("regions") +if regions: + regions_df = pd.DataFrame(regions) + st.dataframe(regions_df) +else: + st.info("No workload hosts found.") diff --git a/streamlit_server/views/details.py b/streamlit_server/views/details.py deleted file mode 100644 index 0192deb..0000000 --- a/streamlit_server/views/details.py +++ /dev/null @@ -1,208 +0,0 @@ -import streamlit as st -import pandas as pd -from streamlit_server.utils.helpers import format_timestamp - -def render(): - resource = st.session_state.selected_resource - resource_type = st.session_state.selected_resource_type - - if not resource: - st.warning("No resource selected.") - return - - print(f"Resource type: {resource_type}") - - if resource_type == 'universe': - render_detail_view_universe(resource) - elif resource_type == 'project': - render_detail_view_project(resource) - elif resource_type == 'vdc': - render_detail_view_vdc(resource) - elif resource_type == 'region': - render_detail_view_region(resource) - elif resource_type == 'network': - render_detail_view_network(resource) - elif resource_type == 'container': - render_detail_view_container(resource) - -def render_detail_view_container(container): - - """ - Renders the details of a specific container workload. - """ - container = st.session_state.client.get_container(container['id']) - if not container: - st.error("Container workload not found.") - return - - st.title(f"Container Workload: {container['container_id']}") - - st.header("Container Workload Details") - col1, col2 = st.columns(2) - with col1: - st.markdown(f"**ID:** `{container['id']}`") - st.markdown(f"**Created:** {format_timestamp(container.get('created_at', 'Unknown'))}") - st.markdown(f"**VDC ID:** {container['vdc_id']}") - st.markdown(f"**Workload Host ID:** {container.get('container_host_id', 'N/A')}") - with col2: - if container.get('updated_at'): - st.markdown(f"**Last Updated:** {format_timestamp(container['updated_at'])}") - st.markdown(f"**Status:** {container['status']}") - st.markdown(f"**Launch Parameters:** {container.get('launch_params', 'N/A')}") - - # Edit Container Workload Form - with st.expander("Edit Container Workload"): - with st.form("edit_container"): - new_status = st.selectbox("Status", ["running", "stopped", "pending","failed-deleted","pending-deleted","pending-allocation","failed-allocation","pending-allocated"], index=["running", "stopped", "pending","failed-deleted","pending-deleted","pending-allocation","failed-allocation","pending-allocated"].index(container['status'])) - new_launch_params = st.text_input("Launch Parameters", value=container.get('launch_params', '')) - submit = st.form_submit_button("Update Container Workload") - - if submit: - result = st.session_state.client.edit_container(container['id'], { - "status": new_status, - "launch_params": new_launch_params - }) - if result: - st.success("Container workload updated successfully!") - st.session_state.selected_resource = container - st.session_state.view_type = 'detail' - st.session_state.selected_resource_type = 'container' - st.rerun() - - # Delete Container Workload Button - if st.button("Delete Container Workload"): - result = st.session_state.client.delete_container(container['id']) - if result: - st.success("Container workload deleted successfully!") - st.session_state.selected_resource = None - st.session_state.view_type = 'list' - st.session_state.selected_resource_type = 'container' - st.rerun() - -def render_detail_view_universe(universe): - st.title(f"Universe: {universe['name']}") - - st.header("Universe Details") - col1, col2 = st.columns(2) - with col1: - st.markdown(f"**ID:** `{universe['id']}`") - st.markdown(f"**Created:** {format_timestamp(universe['created_at'])}") - with col2: - if universe.get('updated_at'): - st.markdown(f"**Last Updated:** {format_timestamp(universe['updated_at'])}") - if universe.get('description'): - st.markdown(f"**Description:** {universe['description']}") - -def render_detail_view_project(project): - st.title(f"Project: {project['name']}") - - st.header("Project Details") - col1, col2 = st.columns(2) - with col1: - st.markdown(f"**ID:** `{project['id']}`") - st.markdown(f"**Created:** {format_timestamp(project['created_at'])}") - with col2: - st.markdown(f"**Universe ID:** `{project['universe_id']}`") - if project.get('updated_at'): - st.markdown(f"**Last Updated:** {format_timestamp(project['updated_at'])}") - -def render_detail_view_vdc(vdc): - st.title(f"Virtual Data Center: {vdc['name']}") - - st.header("VDC Details") - col1, col2 = st.columns(2) - with col1: - st.markdown(f"**ID:** `{vdc['id']}`") - st.markdown(f"**Created:** {format_timestamp(vdc['created_at'])}") - with col2: - st.markdown(f"**Project ID:** `{vdc['project_id']}`") - if vdc.get('updated_at'): - st.markdown(f"**Last Updated:** {format_timestamp(vdc['updated_at'])}") - - -def render_detail_view_network(network): - st.title(f"Network: {network['name']}") - - st.header("Network Details") - col1, col2 = st.columns(2) - with col1: - st.markdown(f"**ID:** `{network['id']}`") - st.markdown(f"**Created:** {network.get('created_at', 'Unknown')}") - st.markdown(f"**IPv4 CIDR:** {network.get('ipv4_cidr', 'N/A')}") - st.markdown(f"**IPv4 Gateway:** {network.get('ipv4_gateway', 'N/A')}") - st.markdown(f"**Encapsulation Type:** {network.get('encapsulation', 'N/A')}") - st.markdown(f"**VNI:** {network.get('vni', 'N/A')}") - with col2: - if network.get('updated_at'): - st.markdown(f"**Last Updated:** {network['updated_at']}") - if network.get('description'): - st.markdown(f"**Description:** {network['description']}") - st.markdown(f"**IPv6 CIDR:** {network.get('ipv6_cidr', 'N/A')}") - st.markdown(f"**IPv6 Gateway:** {network.get('ipv6_gateway', 'N/A')}") - - st.header("Network Ports") - network_ports = st.session_state.client.get_network_ports_by_network(network['id']) - - if network_ports: - # Fetch workloads to map workload_id to workload name - # TODO Enable to fetching of workloads once htey exist - # workloads = st.session_state.client.get_workloads() - # workload_map = {workload['id']: workload['name'] for workload in workloads} - - # Add workload names to each network port entry - for port in network_ports: - port['workload_name'] = "Unk" #workload_map.get(port['workload_id'], 'Unknown Workload') - - # Create a DataFrame with the enriched data - df = pd.DataFrame(network_ports) - df = df[[ - 'id', 'workload_id', 'workload_name', 'ip_address', 'mac_address', - 'port_type', 'state' - ]] - st.dataframe(df, use_container_width=True) - else: - st.info("No network ports found for this network.") - - - - -def render_detail_view_region(region): - st.title(f"Region: {region['name']}") - - st.header("Region Details") - col1, col2 = st.columns(2) - with col1: - st.markdown("**Region ID**") - st.code(f"{region['id']}") - st.markdown("**Region Enrollment Key**(Confidental)") - st.code(f"{region['enrollment_key']}") - with col2: - st.markdown(f"**Last Updated:** {region['updated_at']}") - st.markdown(f"**Created:** {region.get('created_at', 'Unknown')}") - - if region.get('description'): - st.markdown(f"**Description:** {region['description']}") - - st.header("Region Access Entries") - region_access = st.session_state.client.get_region_access(region['id']) - - if region_access: - # Fetch all projects and regions - projects = st.session_state.client.get_projects() - regions = st.session_state.client.get_regions() - - # Create dictionaries to map IDs to names - project_map = {project['id']: project['name'] for project in projects} - region_map = {region['id']: region['name'] for region in regions} - - # Add project and region names to each entry - for entry in region_access: - entry['project_name'] = project_map.get(entry['project_id'], 'Unknown Project') - entry['region_name'] = region_map.get(entry['region_id'], 'Unknown Region') - - # Create a DataFrame with the enriched data - df = pd.DataFrame(region_access) - df = df[['project_id', 'project_name', 'region_id', 'region_name']] # Include names in the table - st.dataframe(df, use_container_width=True) - else: - st.info("No access entries found for this region.") \ No newline at end of file diff --git a/streamlit_server/views/hosts.py b/streamlit_server/views/hosts.py index 54300e8..66b75cf 100644 --- a/streamlit_server/views/hosts.py +++ b/streamlit_server/views/hosts.py @@ -2,16 +2,16 @@ import streamlit as st import pandas as pd from streamlit_server.utils.helpers import format_timestamp -def render(): - st.header("Worker & Host Management") - - # Fetch Worker Agents & Workload Hosts - workload_hosts = st.session_state.client.get_workload_hosts() - - # Display Workload Hosts - st.subheader("Workload Hosts") - if workload_hosts: - workload_hosts_df = pd.DataFrame(workload_hosts) - st.dataframe(workload_hosts_df) - else: - st.info("No workload hosts found.") +# def render(): +st.header("Worker & Host Management") + +# Fetch Worker Agents & Workload Hosts +workload_hosts = st.session_state.client.get_workload_hosts() + +# Display Workload Hosts +st.subheader("Workload Hosts") +if workload_hosts: + workload_hosts_df = pd.DataFrame(workload_hosts) + st.dataframe(workload_hosts_df) +else: + st.info("No workload hosts found.") diff --git a/streamlit_server/views/images.py b/streamlit_server/views/images.py index b41a847..9430bd4 100644 --- a/streamlit_server/views/images.py +++ b/streamlit_server/views/images.py @@ -1,32 +1,34 @@ import streamlit as st +import pandas as pd from streamlit_server.utils.helpers import format_timestamp -def render_image_list(): - """ - Renders a list of all images. - """ +def render_list(): st.header("Image Management") # Create Image Form with st.expander("Create New Image"): with st.form("create_image"): - location = st.text_input("Location (URL or Path)") - size = st.number_input("Size (in MB)", min_value=0.0) - location_type = st.selectbox("Location Type", ["url", "local"]) - os_family = st.text_input("OS Family") - os_version = st.text_input("OS Version") + name = st.text_input("Image Name") + location = st.text_input("Image Location (e.g., /images/image.img)") + location_type = st.selectbox("Location Type", ["Local", "Remote"]) + size = st.number_input("Size (GB)", min_value=0.1, step=0.1) + os_family = st.text_input("OS Family (e.g., Linux)") + os_version = st.text_input("OS Version (e.g., 20.04)") + checksum = st.text_input("Checksum (optional)") submit = st.form_submit_button("Create Image") - if submit and location and size: - result = st.session_state.client.create_image({ - "location": location, - "size": size, - "location_type": location_type, - "os_family": os_family, - "os_version": os_version - }) + if submit and name and location: + result = st.session_state.client.create_image( + name=name, + location=location, + location_type=location_type, + size=size, + os_family=os_family, + os_version=os_version, + checksum=checksum + ) if result: - st.success(f"Image created successfully!") + st.success(f"Image '{name}' created successfully!") # List Images st.subheader("Existing Images") @@ -37,93 +39,45 @@ def render_image_list(): with st.container(): col1, col2, col3 = st.columns([3, 1, 1]) with col1: - st.markdown(f"### {image['location']}") - st.text(f"Size: {image['size']} MB") - st.text(f"OS: {image.get('os_family', 'N/A')} {image.get('os_version', 'N/A')}") + st.markdown(f"### {image['name']}") + st.text(f"Location: {image['location']}") + st.text(f"Size: {image['size']} GB") with col2: st.text(f"ID: {image['id'][:8]}...") with col3: if st.button("View Details", key=f"view_{image['id']}"): st.session_state.selected_resource = image st.session_state.view_type = 'detail' - st.session_state.selected_resource_type='image' + st.session_state.selected_resource_type = 'image' st.rerun() # Force Streamlit to rerun the script - else: - st.info("No images found.") -def render_image_detail(image_id): - """ - Renders the details of a specific image. - """ - image = st.session_state.client.get_image(image_id) - if not image: - st.error("Image not found.") - return +def render_detail(): + image = st.session_state.client.get_image(st.session_state.selected_resource['id']) - st.title(f"Image: {image['location']}") + st.title(f"Image: {image['name']}") + if st.button("Back to list view"): + st.session_state.view_type = 'list' + st.rerun() st.header("Image Details") col1, col2 = st.columns(2) with col1: st.markdown(f"**ID:** `{image['id']}`") - st.markdown(f"**Created:** {format_timestamp(image.get('created_at', 'Unknown'))}") - st.markdown(f"**Size:** {image['size']} MB") - st.markdown(f"**Location Type:** {image.get('location_type', 'N/A')}") + st.markdown(f"**Created:** {format_timestamp(image['created_at'])}") + st.markdown(f"**Location:** `{image['location']}`") + st.markdown(f"**Location Type:** `{image['location_type']}`") with col2: + st.markdown(f"**Size:** `{image['size']} GB`") + st.markdown(f"**OS Family:** `{image['os_family']}`") + st.markdown(f"**OS Version:** `{image['os_version']}`") + if image.get('checksum'): + st.markdown(f"**Checksum:** `{image['checksum']}`") if image.get('updated_at'): st.markdown(f"**Last Updated:** {format_timestamp(image['updated_at'])}") - st.markdown(f"**OS Family:** {image.get('os_family', 'N/A')}") - st.markdown(f"**OS Version:** {image.get('os_version', 'N/A')}") - - # Edit Image Form - with st.expander("Edit Image"): - with st.form("edit_image"): - new_location = st.text_input("Location (URL or Path)", value=image['location']) - new_size = st.number_input("Size (in MB)", min_value=0.0, value=image['size']) - new_location_type = st.selectbox("Location Type", ["url", "local"], index=0 if image.get('location_type') == "url" else 1) - new_os_family = st.text_input("OS Family", value=image.get('os_family', '')) - new_os_version = st.text_input("OS Version", value=image.get('os_version', '')) - submit = st.form_submit_button("Update Image") - - if submit: - result = st.session_state.client.edit_image(image['id'], { - "location": new_location, - "size": new_size, - "location_type": new_location_type, - "os_family": new_os_family, - "os_version": new_os_version - }) - if result: - st.success("Image updated successfully!") - st.session_state.selected_resource = image - st.session_state.view_type = 'detail' - st.session_state.selected_resource_type='image' - - # Delete Image Button - if st.button("Delete Image"): - result = st.session_state.client.delete_image(image['id']) - if result: - st.success("Image deleted successfully!") - st.session_state.view_type = 'list' - st.session_state.selected_resource_type='image' - -def render(): - """ - Main render function for the Images page. - """ - # Read query parameters for routing - if not st.session_state.get('api_key'): - st.warning("Please enter your API key in the sidebar to continue.") - return - - if st.session_state.view_type == 'list': - print("Viewing list") - render_image_list() - elif st.session_state.view_type == 'detail' and st.session_state.selected_resource: - print("Viewing detail") - render_image_detail(st.session_state.selected_resource["id"]) - else: - st.error("Invalid view type or resource ID.") \ No newline at end of file +if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type == 'image': + render_detail() +else: + render_list() \ No newline at end of file diff --git a/streamlit_server/views/networks.py b/streamlit_server/views/networks.py index d263364..eeb7c7d 100644 --- a/streamlit_server/views/networks.py +++ b/streamlit_server/views/networks.py @@ -2,14 +2,14 @@ import streamlit as st import pandas as pd from streamlit_server.utils.helpers import format_timestamp -def render(): +def render_list(): st.header("Network Management") # Create Network Form with st.expander("Create New Network"): with st.form("create_network"): name = st.text_input("Network Name") - vdcs = st.session_state.client.get_vdcs() + vdcs = st.session_state.client.get_virtual_data_centers() vdc_choices = {v['name']: v['id'] for v in vdcs} if vdcs else {} selected_vdc = st.selectbox("Select VDC", list(vdc_choices.keys())) submit = st.form_submit_button("Create Network") @@ -38,3 +38,63 @@ def render(): st.session_state.view_type = 'detail' st.session_state.selected_resource_type='network' st.rerun() # Force Streamlit to rerun the script + + + +def render_detail(): + network = st.session_state.client.get_network(st.session_state.selected_resource['id']) + if not network: + st.error("Network not found.") + st.session_state.selected_resource = None + st.session_state.view_type = 'list' + st.session_state.selected_resource_type = None + return + + st.title(f"Network: {network['name']}") + if st.button("Back to list view"): + st.session_state.view_type = 'list' + st.rerun() + + st.header("Network Details") + col1, col2 = st.columns(2) + with col1: + st.markdown(f"**ID:** `{network['id']}`") + st.markdown(f"**VDC ID:** `{network['vdc_id']}`") + st.markdown(f"**Created:** {network.get('created_at', 'Unknown')}") + st.markdown(f"**IPv4 CIDR:** {network.get('ipv4_cidr', 'N/A')}") + st.markdown(f"**IPv4 Gateway:** {network.get('ipv4_gateway', 'N/A')}") + with col2: + if network.get('updated_at'): + st.markdown(f"**Last Updated:** {network['updated_at']}") + if network.get('description'): + st.markdown(f"**Description:** {network['description']}") + st.markdown(f"**IPv6 CIDR:** {network.get('ipv6_cidr', 'N/A')}") + st.markdown(f"**IPv6 Gateway:** {network.get('ipv6_gateway', 'N/A')}") + + st.header("Network Ports") + network_ports = st.session_state.client.get_network_ports_by_network(network['id']) + + if network_ports: + # Fetch workloads to map workload_id to workload name + # TODO Enable to fetching of workloads once htey exist + # workloads = st.session_state.client.get_workloads() + # workload_map = {workload['id']: workload['name'] for workload in workloads} + + # Add workload names to each network port entry + for port in network_ports: + port['workload_name'] = "Unk" #workload_map.get(port['workload_id'], 'Unknown Workload') + + # Create a DataFrame with the enriched data + df = pd.DataFrame(network_ports) + df = df[[ + 'id', 'workload_id', 'workload_name', 'ip_address', 'mac_address', + 'port_type' + ]] + st.dataframe(df, use_container_width=True) + else: + st.info("No network ports found for this network.") + +if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='network': + render_detail() +else: + render_list() \ No newline at end of file diff --git a/streamlit_server/views/projects.py b/streamlit_server/views/projects.py index 55d6c54..b6ff9b5 100644 --- a/streamlit_server/views/projects.py +++ b/streamlit_server/views/projects.py @@ -2,7 +2,7 @@ import streamlit as st import pandas as pd from streamlit_server.utils.helpers import format_timestamp -def render(): +def render_list(): st.header("Project Management") # Create Project Form @@ -39,3 +39,27 @@ def render(): st.session_state.selected_resource_type='project' st.rerun() # Force Streamlit to rerun the script + +def render_detail(): + project = st.session_state.client.get_project(st.session_state.selected_resource['id']) + + st.title(f"Project: {project['name']}") + if st.button("Back to list view"): + st.session_state.view_type = 'list' + st.rerun() + + st.header("Project Details") + col1, col2 = st.columns(2) + with col1: + st.markdown(f"**ID:** `{project['id']}`") + st.markdown(f"**Created:** {format_timestamp(project['created_at'])}") + with col2: + st.markdown(f"**Universe ID:** `{project['universe_id']}`") + if project.get('updated_at'): + st.markdown(f"**Last Updated:** {format_timestamp(project['updated_at'])}") + + +if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='project': + render_detail() +else: + render_list() \ No newline at end of file diff --git a/streamlit_server/views/regions.py b/streamlit_server/views/regions.py index 2432da9..28e993b 100644 --- a/streamlit_server/views/regions.py +++ b/streamlit_server/views/regions.py @@ -2,7 +2,7 @@ import streamlit as st import pandas as pd from streamlit_server.utils.helpers import format_timestamp -def render(): +def render_list(): st.header("Region Management") # Create Region Form @@ -39,3 +39,53 @@ def render(): st.session_state.selected_resource_type='region' st.rerun() # Force Streamlit to rerun the script + + +def render_detail(): + region = st.session_state.client.get_region(st.session_state.selected_resource['id']) + + st.title(f"Region: {region['name']}") + if st.button("Back to list view"): + st.session_state.view_type = 'list' + st.rerun() + + st.header("Region Details") + col1, col2 = st.columns(2) + with col1: + st.code(f"{region['id']}") + st.markdown(f"**Created:** {region.get('created_at', 'Unknown')}") + with col2: + if region.get('updated_at'): + st.markdown(f"**Last Updated:** {region['updated_at']}") + if region.get('description'): + st.markdown(f"**Description:** {region['description']}") + + st.header("Region Access Entries") + region_access = st.session_state.client.get_region_access(region['id']) + + if region_access: + # Fetch all projects and regions + projects = st.session_state.client.get_projects() + regions = st.session_state.client.get_regions() + + # Create dictionaries to map IDs to names + project_map = {project['id']: project['name'] for project in projects} + region_map = {region['id']: region['name'] for region in regions} + + # Add project and region names to each entry + for entry in region_access: + entry['project_name'] = project_map.get(entry['project_id'], 'Unknown Project') + entry['region_name'] = region_map.get(entry['region_id'], 'Unknown Region') + + # Create a DataFrame with the enriched data + df = pd.DataFrame(region_access) + df = df[['project_id', 'project_name', 'region_id', 'region_name']] # Include names in the table + st.dataframe(df, use_container_width=True) + else: + st.info("No access entries found for this region.") + + +if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='region': + render_detail() +else: + render_list() \ No newline at end of file diff --git a/streamlit_server/views/sidebar.py b/streamlit_server/views/sidebar.py index 7d60e34..e4b4115 100644 --- a/streamlit_server/views/sidebar.py +++ b/streamlit_server/views/sidebar.py @@ -7,31 +7,77 @@ def render(): if 'api_key' not in st.session_state or st.session_state.api_key == "": st.session_state.api_key = "xxx" print("Providing xxx as api key") + # Display the API key input with the default value api_key = st.sidebar.text_input("API Key", type="password", value=st.session_state.get('api_key')) if api_key: st.session_state.api_key = api_key + # Define views with appropriate icons + views = { + "Universes": "🌌", + "Projects": "📂", + "Virtual Data Centers": "🏢", + "Networks": "🔌", + "Regions": "🌎", + "Hosts": "💻", + "Images": "🖼️", + "Containers": "📦", + "Websocket": "🔄", + "Virtual Machines": "⚙️" + } + + # Style the sidebar navigation + st.sidebar.markdown(""" + + """, unsafe_allow_html=True) + st.sidebar.header("Navigation") - views = ["Universes", "Projects", "Virtual Data Centers", "Networks", "Regions", "Hosts", "Images", "Containers", "Websocket"] - + + # Initialize current view in session state if not present + if 'current_view' not in st.session_state: + st.session_state.current_view = list(views.keys())[0] + + # Format the options with icons + format_func = lambda x: f"{views[x]} {x}" + + # Use selectbox for navigation if st.session_state.view_type == 'list': - # if st.sidebar.radio("Select View", views): - # print(f"Radio buton pressed") - # st.session_state.current_view = "D" - for _viewType in views: - if st.sidebar.button(_viewType): - print(f"Processing sidebar button pres {_viewType}") - st.session_state.current_view=_viewType - st.session_state.selected_resource = None - st.session_state.view_type = 'list' - st.session_state.selected_resource_type = None - # st.rerun() # Force Streamlit to rerun the script + selected_view = st.sidebar.selectbox( + "Select Resource Type", + options=list(views.keys()), + format_func=format_func, + index=list(views.keys()).index(st.session_state.current_view), + label_visibility="collapsed" + ) + + # Handle navigation + if selected_view != st.session_state.current_view: + st.session_state.current_view = selected_view + st.session_state.selected_resource = None + st.session_state.view_type = 'list' + st.session_state.selected_resource_type = None + st.rerun() if st.session_state.view_type == 'detail': if st.sidebar.button("← Back to List View"): st.session_state.view_type = 'list' st.session_state.selected_resource = None st.session_state.selected_resource_type = None - st.rerun() # Force Streamlit to rerun the script - + st.rerun() \ No newline at end of file diff --git a/streamlit_server/views/universes.py b/streamlit_server/views/universes.py index 72e3ca1..e78e0d3 100644 --- a/streamlit_server/views/universes.py +++ b/streamlit_server/views/universes.py @@ -2,7 +2,7 @@ import streamlit as st import pandas as pd from streamlit_server.utils.helpers import format_timestamp -def render(): +def render_list(): st.header("Universe Management") with st.expander("Create New Universe"): @@ -37,3 +37,33 @@ def render(): st.rerun() # Force Streamlit to rerun the script + +def render_detail(): + universe = st.session_state.client.get_universe(st.session_state.selected_resource['id']) + if not universe: + st.error("Universe not found.") + st.session_state.selected_resource = None + st.session_state.view_type = 'list' + st.session_state.selected_resource_type = None + return + + st.title(f"Universe: {universe['name']}") + if st.button("Back to list view"): + st.session_state.view_type = 'list' + st.rerun() + + st.header("Universe Details") + col1, col2 = st.columns(2) + with col1: + st.markdown(f"**ID:** `{universe['id']}`") + st.markdown(f"**Created:** {format_timestamp(universe['created_at'])}") + with col2: + if universe.get('updated_at'): + st.markdown(f"**Last Updated:** {format_timestamp(universe['updated_at'])}") + if universe.get('description'): + st.markdown(f"**Description:** {universe['description']}") + +if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='universe': + render_detail() +else: + render_list() \ No newline at end of file diff --git a/streamlit_server/views/vdcs.py b/streamlit_server/views/virtual_data_centers.py similarity index 57% rename from streamlit_server/views/vdcs.py rename to streamlit_server/views/virtual_data_centers.py index c19d6ea..71c7928 100644 --- a/streamlit_server/views/vdcs.py +++ b/streamlit_server/views/virtual_data_centers.py @@ -2,13 +2,13 @@ import streamlit as st import pandas as pd from streamlit_server.utils.helpers import format_timestamp -def render(): - st.header("Virtual Data Center (VDC) Management") +def render_list(): + st.header("Virtual Data Center Management") # Create VDC Form - with st.expander("Create New VDC"): + with st.expander("Create New Virtual Data Center"): with st.form("create_vdc"): - name = st.text_input("VDC Name") + name = st.text_input("Virtual Data Center Name") projects = st.session_state.client.get_projects() project_choices = {p['name']: p['id'] for p in projects} if projects else {} selected_project = st.selectbox("Select Project", list(project_choices.keys())) @@ -17,11 +17,11 @@ def render(): if submit and name and selected_project: result = st.session_state.client.create_vdc(name, project_choices[selected_project]) if result: - st.success(f"VDC '{name}' created successfully!") + st.success(f"Virtual Data Center '{name}' created successfully!") # List VDCs - st.subheader("Existing VDCs") - vdcs = st.session_state.client.get_vdcs() + st.subheader("Existing Virtual Data Center") + vdcs = st.session_state.client.get_virtual_data_centers() # projects = st.session_state.client.get_projects() # regions = st.session_state.client.get_regions() @@ -41,7 +41,31 @@ def render(): if st.button("View Details", key=f"view_{vdc['id']}"): st.session_state.selected_resource = vdc st.session_state.view_type = 'detail' - st.session_state.selected_resource_type='vdc' + st.session_state.selected_resource_type='virtual_data_center' st.rerun() # Force Streamlit to rerun the script + +def render_detail(): + + vdc = st.session_state.client.get_virtual_data_center(st.session_state.selected_resource['id']) + st.title(f"Virtual Data Center: {vdc['name']}") + + if st.button("Back to list view"): + st.session_state.view_type = 'list' + st.rerun() + + st.header("Virtual Data Center Details") + col1, col2 = st.columns(2) + with col1: + st.markdown(f"**ID:** `{vdc['id']}`") + st.markdown(f"**Created:** {format_timestamp(vdc['created_at'])}") + with col2: + st.markdown(f"**Project ID:** `{vdc['project_id']}`") + if vdc.get('updated_at'): + st.markdown(f"**Last Updated:** {format_timestamp(vdc['updated_at'])}") + +if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='virtual_data_center': + render_detail() +else: + render_list() \ No newline at end of file diff --git a/streamlit_server/views/virtual_machines.py b/streamlit_server/views/virtual_machines.py new file mode 100644 index 0000000..be9d18a --- /dev/null +++ b/streamlit_server/views/virtual_machines.py @@ -0,0 +1,288 @@ +import json +import streamlit as st +from streamlit_server.utils.helpers import format_timestamp + + +def render_list(): + """ + Renders a list of all VM workloads. + """ + st.header("VM Workload Management") + + # Initialize session state variables for delete confirmation + if 'delete_confirmation_shown' not in st.session_state: + st.session_state.delete_confirmation_shown = False + if 'delete_confirmed' not in st.session_state: + st.session_state.delete_confirmed = False + + # Fetch all VDCs for the dropdown + vdcs = st.session_state.client.get_virtual_data_centers() + vdc_options = {vdc['name']: vdc['id'] for vdc in vdcs} if vdcs else {} + + # Add a dropdown to select VDC + selected_vdc_name = st.selectbox("Filter by Virtual Data Center", list(vdc_options.keys())) + + # Fetch all Workload Hosts for the dropdown + workload_hosts = st.session_state.client.get_workload_hosts() + workload_host_options = {workload_host['hostname']: workload_host['id'] for workload_host in workload_hosts} if workload_hosts else {} + + # Create VM Workload Form + with st.expander("Create New VM Workload"): + with st.form("create_virtual_machine"): + # Dropdown for VDC selection + selected_vdc_name_create = st.selectbox("VDC", list(vdc_options.keys())) + vm_name = st.text_input("VM Name", value="my-vm") + memory = st.number_input("Memory (MB)", min_value=512, value=2048) + vcpu = st.number_input("vCPU", min_value=1, value=2) + + # Networks Dropdown + networks = st.session_state.client.get_networks() + network_options = {net['id']: net['name'] for net in networks} + selected_network_ids = st.multiselect("Networks", options=list(network_options.keys()), format_func=lambda x: network_options[x]) + + # Volumes Section + st.subheader("Volumes") + if 'volumes' not in st.session_state: + st.session_state.volumes = [] + + for i, volume in enumerate(st.session_state.volumes): + col1, col2, col3, col4 = st.columns([3, 2, 2, 2]) + + with col1: + volume_name = st.text_input(f"Volume Name {i+1}", value=volume['name'], key=f"volume_name_{i}") + + with col2: + volume_size = st.number_input(f"Size (GB) {i+1}", min_value=1, value=volume['size_gb'], key=f"volume_size_{i}") + + with col3: + volume_type = st.selectbox(f"Type {i+1}", options=["local", "ceph"], index=0 if volume.get('type', 'local') == 'local' else 1, key=f"volume_type_{i}") + + with col4: + # Get image options + image_list = st.session_state.client.get_images() + + # Create a mapping of image names to IDs + image_name_to_id = {img['name']: img['id'] for img in image_list} + image_name_to_id["blank"] = "" # Add a blank option + + # Default selected image name (fallback to 'blank' if not found) + default_image_name = volume.get('image_name', 'blank') + + # Dropdown to display image names + selected_image_name = st.selectbox( + f"Image {i+1}", + options=list(image_name_to_id.keys()), + index=list(image_name_to_id.keys()).index(default_image_name) if default_image_name in image_name_to_id else 0, + key=f"volume_image_{i}" + ) + + # Get the corresponding image ID for the selected image name + selected_image_id = image_name_to_id[selected_image_name] + + # Store updated volume info + if selected_image_name == "blank": + st.session_state.volumes[i] = {"name": volume_name, "size_gb": volume_size, "type": volume_type} + else: + st.session_state.volumes[i] = {"name": volume_name, "size_gb": volume_size, "type": volume_type, "image_id": selected_image_id} + + submit = st.form_submit_button("Create VM Workload") + + # **Move the buttons OUTSIDE of the form** + col1, col2 = st.columns([1, 1]) + with col1: + if st.button("Add Volume"): + st.session_state.volumes.append({"name": "", "size_gb": 10, "type": "", "image":""}) + st.rerun() + + with col2: + for i in range(len(st.session_state.volumes)): + if st.button(f"Remove Volume {i+1}", key=f"remove_volume_{i}"): + st.session_state.volumes.pop(i) + st.rerun() + + if submit and selected_vdc_name_create and vm_name: + # Use the selected VDC's ID + vdc_id = vdc_options[selected_vdc_name_create] + + result = st.session_state.client.create_virtual_machine({ + "virtual_data_center": vdc_id, + "virtual-machines": [{ + "name": vm_name, + "memory": memory, + "vcpu": vcpu, + "volumes": st.session_state.volumes, + "networks": [{"id": net_id} for net_id in selected_network_ids] + }] + }) + if result: + st.success("VM workload created successfully!") + st.session_state.volumes = [] # Reset volumes after creation + st.rerun() + + # List Container Workloads + st.subheader("Existing Container Workloads") + VirtualMachines = st.session_state.client.get_virtual_machines() + print(VirtualMachines) + if VirtualMachines: + # Filter VirtualMachines based on the selected VDC + if selected_vdc_name: + selected_vdc_id = vdc_options[selected_vdc_name] + filtered_VirtualMachines = [VirtualMachine for VirtualMachine in VirtualMachines if VirtualMachine['vdc_id'] == selected_vdc_id] + else: + filtered_VirtualMachines = VirtualMachines + + # Delete All functionality + if not st.session_state.delete_confirmation_shown: + if st.button("Delete All VMs", key="delete_all_vms", type="primary", help="This will delete all visible virtual machines"): + st.session_state.delete_confirmation_shown = True + st.rerun() + + # Show confirmation if delete was clicked + if st.session_state.delete_confirmation_shown and not st.session_state.delete_confirmed: + st.warning("Are you sure you want to delete all virtual machines? This action cannot be undone.") + confirm_col1, confirm_col2 = st.columns([1, 1]) + with confirm_col1: + if st.button("Yes, Delete All", key="confirm_delete_all"): + st.session_state.delete_confirmed = True + st.rerun() + with confirm_col2: + if st.button("Cancel", key="cancel_delete_all"): + st.session_state.delete_confirmation_shown = False + st.rerun() + + # Perform deletion if confirmed + if st.session_state.delete_confirmed: + deletion_count = 0 + for vm in filtered_VirtualMachines: + # Call the delete API for each VM + result = st.session_state.client.delete_virtual_machine(vm['id']) + if result: + deletion_count += 1 + + st.success(f"Successfully deleted {deletion_count} virtual machines.") + # Reset the confirmation flags + st.session_state.delete_confirmation_shown = False + st.session_state.delete_confirmed = False + filtered_VirtualMachines=[] + + # Display the VMs + for VirtualMachine in filtered_VirtualMachines: + with st.container(): + col1, col2, col3 = st.columns([3, 1, 1]) + with col1: + # Safely handle empty or None launch_params + launch_params = VirtualMachine.get('launch_params') + if launch_params: + try: + VirtualMachine_name = json.loads(launch_params).get('docker_image') + except json.JSONDecodeError: + print("LP not valid json") + VirtualMachine_name = None + else: + print("lp empty") + VirtualMachine_name = None + st.markdown(f"### {VirtualMachine['name']}") + + # Create a grey box for the status + status = VirtualMachine['status'] + if status == "error": + status_color = "red" + elif status == "running": + status_color = "green" + elif status == "pending-allocated": + status_color = "orange" + else: + status_color = "blue" + + st.markdown( + f""" +
+ Status: {status} +
+ """, + unsafe_allow_html=True + ) + + st.text(f"VDC ID: {VirtualMachine['vdc_id']}") + st.text(f"Image: {VirtualMachine_name}") + with col2: + st.text(f"ID: {VirtualMachine['id']}") + with col3: + if st.button("View Details", key=f"view_{VirtualMachine['id']}"): + print("Button is pressed") + st.session_state.selected_resource = VirtualMachine + st.session_state.view_type = 'detail' + st.session_state.selected_resource_type = 'VirtualMachine' + st.rerun() + + else: + st.info("No VirtualMachine workloads found.") + + + + + +def render_detail(): + """ + Renders the details of a specific VM workload. + """ + VirtualMachine = st.session_state.client.get_virtual_machine(st.session_state.selected_resource['id']) + if not VirtualMachine: + st.error("VM workload not found.") + return + + st.title(f"VM Workload: {VirtualMachine['name']}") + if st.button("Back to list view"): + st.session_state.view_type = 'list' + st.rerun() + + st.header("VM Workload Details") + col1, col2 = st.columns(2) + with col1: + st.markdown(f"**ID:** `{VirtualMachine['id']}`") + st.markdown(f"**Created:** {format_timestamp(VirtualMachine.get('created_at', 'Unknown'))}") + st.markdown(f"**VDC ID:** {VirtualMachine['vdc_id']}") + st.markdown(f"**Workload Host ID:** {VirtualMachine.get('workload_host_id', 'N/A')}") + with col2: + if VirtualMachine.get('updated_at'): + st.markdown(f"**Last Updated:** {format_timestamp(VirtualMachine['updated_at'])}") + st.markdown(f"**Status:** {VirtualMachine['status']}") + st.markdown(f"**Launch Parameters:** {VirtualMachine.get('launch_params', 'N/A')}") + + # Edit VM Workload Form + with st.expander("Edit VM Workload"): + with st.form("edit_vm"): + new_status = st.selectbox("Status", ["running", "stopped", "pending", "failed-deleted", "pending-deleted", "pending-allocation", "failed-allocation", "pending-allocated"], index=["running", "stopped", "pending", "failed-deleted", "pending-deleted", "pending-allocation", "failed-allocation", "pending-allocated"].index(VirtualMachine['status'])) + new_launch_params = st.text_input("Launch Parameters", value=VirtualMachine.get('launch_params', '')) + submit = st.form_submit_button("Update VM Workload") + + if submit: + result = st.session_state.client.edit_virtual_machine(VirtualMachine['id'], { + "status": new_status, + "launch_params": new_launch_params + }) + if result: + st.success("VM workload updated successfully!") + st.session_state.selected_resource = vm + st.session_state.view_type = 'detail' + st.session_state.selected_resource_type = 'VirtualMachine' + st.rerun() + + # Delete VM Workload Button + if st.button("Delete VM Workload"): + result = st.session_state.client.delete_virtual_machine(VirtualMachine['id']) + if result: + st.success("VM workload deleted successfully!") + st.session_state.selected_resource = None + st.session_state.view_type = 'list' + st.session_state.selected_resource_type = 'VirtualMachine' + st.rerun() + + + + + +if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type=='VirtualMachine': + render_detail() +else: + render_list() \ No newline at end of file diff --git a/streamlit_server/views/websocket.py b/streamlit_server/views/websocket.py index 836095b..ea8ac88 100644 --- a/streamlit_server/views/websocket.py +++ b/streamlit_server/views/websocket.py @@ -35,3 +35,5 @@ def render(): st.error("Message cannot be empty!") else: st.warning("No connected clients found.") + +render() \ No newline at end of file diff --git a/streamlit_server/views/workload_hosts.py b/streamlit_server/views/workload_hosts.py new file mode 100644 index 0000000..5b27861 --- /dev/null +++ b/streamlit_server/views/workload_hosts.py @@ -0,0 +1,201 @@ +import streamlit as st +from streamlit_server.utils.helpers import format_timestamp +import uuid + +def render_list(): + st.header("Workload Host Management") + + # Create Workload Host Form + with st.expander("Create New Workload Host"): + with st.form("create_workload_host"): + name = st.text_input("Workload Host Name") + region_id = st.text_input("Region ID") + secret_key = st.text_input("Secret Key", value=str(uuid.uuid4())) + system_manufacturer = st.text_input("System Manufacturer") + system_model = st.text_input("System Model") + physical_identifier = st.text_input("Physical Identifier") + dcim_identifier = st.text_input("DCIM Identifier") + installed_date = st.date_input("Installed Date") + installed_status = st.text_input("Installed Status") + hostname = st.text_input("Hostname") + available_for_scheduling = st.checkbox("Available for Scheduling", value=True) + placement_priority = st.number_input("Placement Priority", value=100) + submit = st.form_submit_button("Create Workload Host") + + if submit and name and region_id: + result = st.session_state.client.create_workload_host( + name=name, + region_id=region_id, + secret_key=secret_key, + system_manufacturer=system_manufacturer, + system_model=system_model, + physical_identifier=physical_identifier, + dcim_identifier=dcim_identifier, + installed_date=installed_date, + installed_status=installed_status, + hostname=hostname, + available_for_scheduling=available_for_scheduling, + placement_priority=placement_priority + ) + if result: + st.success(f"Workload Host '{name}' created successfully!") + + # List Workload Hosts + st.subheader("Existing Workload Hosts") + workload_hosts = st.session_state.client.get_workload_hosts() + + if workload_hosts: + for host in workload_hosts: + with st.container(): + col1, col2, col3 = st.columns([3, 1, 1]) + with col1: + st.markdown(f"### {host['name']}") + st.text(f"Hostname: {host['hostname']}") + st.text(f"Region ID: {host['region_id']}") + with col2: + st.text(f"ID: {host['id'][:8]}...") + with col3: + if st.button("View Details", key=f"view_{host['id']}"): + st.session_state.selected_resource = host + st.session_state.view_type = 'detail' + st.session_state.selected_resource_type = 'workload_host' + st.rerun() # Force Streamlit to rerun the script + +def render_detail(): + host_id = st.session_state.selected_resource['id'] + workload_host = st.session_state.client.get_workload_host(host_id) + + st.title(f"Workload Host: {workload_host['name']}") + if st.button("Back to list view"): + st.session_state.view_type = 'list' + st.rerun() + + st.header("Workload Host Details") + col1, col2 = st.columns(2) + with col1: + st.markdown(f"**ID:** `{workload_host['id']}`") + st.markdown(f"**Hostname:** `{workload_host['hostname']}`") + # Make Region ID clickable + if st.button(f"**Region ID:** `{workload_host['region_id']}`"): + st.session_state.selected_resource = {"id": workload_host['region_id']} + st.session_state.view_type = 'detail' + st.session_state.selected_resource_type = 'region' + st.switch_page("streamlit_server/views/regions.py") + + st.markdown(f"**System Manufacturer:** `{workload_host['system_manufacturer']}`") + st.markdown(f"**System Model:** `{workload_host['system_model']}`") + with col2: + st.markdown(f"**Physical Identifier:** `{workload_host['physical_identifier']}`") + st.markdown(f"**DCIM Identifier:** `{workload_host['dcim_identifier']}`") + st.markdown(f"**Installed Status:** `{workload_host['installed_status']}`") + st.markdown(f"**Available for Scheduling:** `{workload_host['available_for_scheduling']}`") + st.markdown(f"**Placement Priority:** `{workload_host['placement_priority']}`") + + # Display OVS Bridges + st.subheader("Attached OVS Bridges") + ovs_bridges = st.session_state.client.get_ovs_bridges_for_host(host_id) + if ovs_bridges: + for bridge in ovs_bridges: + st.markdown(f"- **{bridge['ovs_bridge_name']}**") + else: + st.info("No OVS bridges attached to this host.") + + # Display Workloads + st.subheader("Assigned Workloads") + workloads = st.session_state.client.get_workloads_for_host(host_id) + + if workloads: + containers = [w for w in workloads if w['workload_type'] == 'container'] + vms = [w for w in workloads if w['workload_type'] == 'virtual_machine'] + + if containers: + st.markdown("### Containers") + # Table header + header_cols = st.columns([3, 2, 2, 2, 2]) + header_cols[0].markdown("**Name**") + header_cols[1].markdown("**ID**") + header_cols[2].markdown("**Status**") + header_cols[3].markdown("**Created At**") + header_cols[4].markdown("**Actions**") + + # Container rows + for container in containers: + cols = st.columns([3, 2, 2, 2, 2]) + cols[0].markdown(container['name']) + cols[1].markdown(f"`{container['id'][:8]}...`") + cols[2].markdown(f"`{container['status']}`") + cols[3].markdown(format_timestamp(container['created_at'])) + + with cols[4]: + with st.popover("⚙️"): + if st.button("View Details", key=f"view_container_{container['id']}"): + st.session_state.selected_resource = container + st.session_state.view_type = 'detail' + st.session_state.selected_resource_type = 'container' + st.switch_page("streamlit_server/views/containers.py") + if st.button("Delete", key=f"delete_container_{container['id']}"): + st.session_state.client.delete_container(container['id']) + st.rerun() + if st.button("Migrate", key=f"migrate_container_{container['id']}"): + st.session_state.client.migrate_workload(container['id'], "NEW_HOST_ID") + st.rerun() + + if vms: + st.markdown("### Virtual Machines") + # Table header + header_cols = st.columns([3, 2, 2, 2, 2]) + header_cols[0].markdown("**Name**") + header_cols[1].markdown("**ID**") + header_cols[2].markdown("**Status**") + header_cols[3].markdown("**Created At**") + header_cols[4].markdown("**Actions**") + + # VM rows + for vm in vms: + cols = st.columns([3, 2, 2, 2, 2]) + cols[0].markdown(vm['name']) + cols[1].markdown(f"`{vm['id'][:8]}...`") + cols[2].markdown(f"`{vm['status']}`") + cols[3].markdown(format_timestamp(vm['created_at'])) + + with cols[4]: + with st.popover("⚙️"): + if st.button("View Details", key=f"view_vm_{vm['id']}"): + st.session_state.selected_resource = vm + st.session_state.view_type = 'detail' + st.session_state.selected_resource_type = 'vm' + st.switch_page("streamlit_server/views/virtual_machines.py") + if st.button("Delete", key=f"delete_vm_{vm['id']}"): + st.session_state.client.delete_virtual_machine(vm['id']) + st.rerun() + if st.button("Migrate", key=f"migrate_vm_{vm['id']}"): + st.session_state.client.migrate_workload(vm['id'], "NEW_HOST_ID") + st.rerun() + + else: + st.info("No virtual machines assigned to this host.") + else: + st.info("No workloads assigned to this host.") + + # Display Fixed Resources + st.subheader("Fixed Resources") + fixed_resources = st.session_state.client.get_fixed_resources_for_host(host_id) + if fixed_resources: + for resource in fixed_resources: + st.markdown(f"- **{resource['type']}** (Model: `{resource['model']}`, Manufacturer: `{resource['manufacturer']}`)") + else: + st.info("No fixed resources attached to this host.") + + # Display Pooled Resources + st.subheader("Pooled Resources") + pooled_resources = st.session_state.client.get_pooled_resources_for_host(host_id) + if pooled_resources: + for resource in pooled_resources: + st.markdown(f"- **{resource['resource_type']}** (Total: `{resource['total_quantity']}`, In Use: `{resource['quantity_in_use']}`, Available: `{resource['quantity_available']}`)") + else: + st.info("No pooled resources attached to this host.") + +if st.session_state.view_type == 'detail' and st.session_state.selected_resource_type == 'workload_host': + render_detail() +else: + render_list() \ No newline at end of file diff --git a/streamlit_server2.py b/streamlit_server2.py new file mode 100644 index 0000000..ea95a5c --- /dev/null +++ b/streamlit_server2.py @@ -0,0 +1,36 @@ +import streamlit as st +from streamlit_server.utils.session import init_session_state + +# Define pages with icons +pages = { + "Your account": [ + st.Page("streamlit_server/views/dashboard.py", title="📊 Dashboard"), + st.Page("streamlit_server/views/hosts.py", title="🖥️ Hosts"), + st.Page("streamlit_server/views/images.py", title="🖼️ Images"), + st.Page("streamlit_server/views/networks.py", title="🌐 Networks"), + st.Page("streamlit_server/views/projects.py", title="📂 Projects"), + st.Page("streamlit_server/views/universes.py", title="🌌 Universes"), + st.Page("streamlit_server/views/regions.py", title="🗺️ Regions"), + st.Page("streamlit_server/views/virtual_data_centers.py", title="🏢 Virtual Data Centers"), + st.Page("streamlit_server/views/websocket.py", title="🔌 Active websockets"), + ], + "Workloads": [ + st.Page("streamlit_server/views/workload_hosts.py", title="🖥️ Workload Hosts"), + st.Page("streamlit_server/views/containers.py", title="📦 Containers"), + st.Page("streamlit_server/views/virtual_machines.py", title="💻 Virtual Machines"), + ], +} + +# Set page config +st.set_page_config( + page_title="IaaS Management Portal", + page_icon="🌐", + layout="wide" +) + +# Initialize session state +init_session_state() + +# Run navigation +pg = st.navigation(pages) +pg.run() \ No newline at end of file diff --git a/websocket_server.py b/websocket_server.py index ee6be09..ac1fc39 100755 --- a/websocket_server.py +++ b/websocket_server.py @@ -474,20 +474,131 @@ def handle_send_message(data): except Exception as e: logger.error(f"Error sending message: {e}") +# @socketio.on("docker_event") +# def handle_docker_event(data): +# # TODO - Somehow we need to validate the authenticity of the worker ID, can we crosscheck the workerID suppied in the payload vs the socket id? +# event_type = data["event_type"] +# container_id = data["container_id"] +# container_name = data["container_name"] +# worker_id = data["worker_id"] +# timestamp = data[""] +# attributes = data["attributes"] + +# # TODO - Figure out what has happened and make an API request back tot he API server updating the releveant resource +# # Pay attention to container deleted events +# # Make sure the API server cross checks the worker ID this event is coming from and the resource ID it's updating + + +@socketio.on("libvirt_event") +def handle_libvirt_event(data): + """ + Handle libvirt events received from the worker. + Extract relevant data and log it to the console. + """ + try: + # Extract relevant data from the event + worker_id = data.get("worker_id") + event_type = data.get("type") + details = data.get("details", {}) + + libvirt_vm_id = details.get("libvirt_vm_id") + system_vm_id = details.get("system_vm_id") + unix_timestamp = details.get("timestamp") + timestamp=datetime.utcfromtimestamp(unix_timestamp).isoformat() + + + # Log the extracted data + logger.info(f"Received libvirt event from worker {worker_id}:") + logger.info(f" Event Type: {event_type}") + logger.info(f" Libvirt VM ID: {libvirt_vm_id}") + logger.info(f" System VM ID: {system_vm_id}") + logger.info(f" Timestamp: {timestamp}") + + # Optionally, log the full details for debugging purposes + logger.debug(f"Full event details: {data}") + + if event_type=="libvirt_started": + #Send the update to the API Server + # system_vm_id + # worker_id + # state="started" + # timestamp + URL=f"/workloads/virtual_machines/status_update/{system_vm_id}" + logger.debug(f"Sending task update payload to API server for VM ID {system_vm_id}") + + payload = { + "new_status": "running", + "system_vm_id": system_vm_id, + "worker_id": worker_id, + "timestamp": timestamp + } + + headers = {"Content-Type": "application/json"} + websocket_server_response = requests.put(f"{api_server_url}/workloads/virtual_machines/status_update/{system_vm_id}", data=json.dumps(payload), headers=headers) + logger.debug( websocket_server_response) + logger.info("Update complete") + + if event_type=="libvirt_stopped": + #Send the update to the API Server + # system_vm_id + # worker_id + # state="started" + # timestamp + URL=f"/workloads/virtual_machines/status_update/{system_vm_id}" + logger.debug(f"Sending task update payload to API server for VM ID {system_vm_id}") + + payload = { + "new_status": "stopped", + "system_vm_id": system_vm_id, + "worker_id": worker_id, + "timestamp": timestamp + } + + headers = {"Content-Type": "application/json"} + websocket_server_response = requests.put(f"{api_server_url}/workloads/virtual_machines/status_update/{system_vm_id}", data=json.dumps(payload), headers=headers) + logger.debug( websocket_server_response) + logger.info("Update complete") + + + except Exception as e: + logger.error(f"Error processing libvirt event: {e}") + @socketio.on("docker_event") def handle_docker_event(data): - # TODO - Somehow we need to validate the authenticity of the worker ID, can we crosscheck the workerID suppied in the payload vs the socket id? - event_type = data["event_type"], - container_id = data["container_id"], - container_name = data["container_name"], - worker_id = data["worker_id"], - timestamp = data[""] - attributes = data["attributes"], - - # TODO - Figure out what has happened and make an API request back tot he API server updating the releveant resource - # Pay attention to container deleted events - # Make sure the API server cross checks the worker ID this event is coming from and the resource ID it's updating - + """ + Handle Docker events received from the worker. + Extract relevant data and log it to the console. + """ + try: + # Extract relevant data from the event + worker_id = data.get("worker_id") + event_type = data.get("type") + details = data.get("details", {}) + + container_id = details.get("container_id") + system_container_id = details.get("system_container_id") + container_name = details.get("container_name") + timestamp = details.get("timestamp") + status = details.get("details", {}).get("status") + image = details.get("details", {}).get("from") + action = details.get("details", {}).get("Action") + + # Log the extracted data + logger.info(f"Received Docker event from worker {worker_id}:") + logger.info(f" Event Type: {event_type}") + logger.info(f" Docker Container ID: {container_id}") + logger.info(f" System Container ID: {system_container_id}") + logger.info(f" Container Name: {container_name}") + logger.info(f" Timestamp: {timestamp}") + logger.info(f" Status: {status}") + logger.info(f" Image: {image}") + logger.info(f" Action: {action}") + + # Optionally, log the full details for debugging purposes + logger.debug(f"Full event details: {data}") + + except Exception as e: + logger.error(f"Error processing Docker event: {e}") @app.route("/api/assign_task", methods=["POST"]) def manually_assign_task(): @@ -546,30 +657,6 @@ def create_task(): except Exception as e: logger.error(f"Error creating task: {e}") return jsonify({"error": "Failed to create task"}), 500 - - # # # Example request structure - # payload_text = { - # "worker_id": "234", - # "task_type": "container", - # "job_details": { - # "tenancyID": "tenant123a", - # "containers": [ - # { - # "container_id": "af47632e-43b7-4474-82fa-475a24fe88d5", - # "docker_image": "nginx", - # "cpu_shares": 1, - # "mem_limit": 128, - # "container_name": "web-server1", - # "command": ["nginx", "-g", "daemon off;"], - # "working_dir": "/usr/share/nginx/html" - # } - # ] - # } - # } - - - - if __name__ == "__main__": diff --git a/worker.py b/worker.py deleted file mode 100644 index ab62c38..0000000 --- a/worker.py +++ /dev/null @@ -1,429 +0,0 @@ -import os -from colorlog import ColoredFormatter -import socketio -import argparse -import logging -import json -import asyncio -import docker - -from worker_tasks.report import ReportTask -from worker_tasks.ping import PingTask -from worker_tasks.file_presence import FilePresenceTask -from worker_tasks.container import ContainerTask - -from dotenv import load_dotenv - -# Configure logging -DEBUG_SOCKETIO=False - -# Custom filter to include the function name -class FunctionNameFilter(logging.Filter): - def filter(self, record): - record.funcName = record.funcName if hasattr(record, 'funcName') else '' - return True - -# Define the colorized log format -log_format = ( - "%(log_color)s%(asctime)s - %(levelname)s - %(funcName)s - %(message)s" -) -date_format = "%Y-%m-%d %H:%M:%S" - -# TODO - Add fucntion name to logging output -# Configure the formatter with colors -formatter = ColoredFormatter( - log_format, - datefmt=date_format, - log_colors={ - "DEBUG": "cyan", - "INFO": "green", - "WARNING": "yellow", - "ERROR": "red", - "CRITICAL": "bold_red", - }, -) - -# Configure the handler -handler = logging.StreamHandler() -handler.setFormatter(formatter) - -# Configure the logger -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logger.addFilter(FunctionNameFilter()) -logger.addHandler(handler) - -# # Ensure the logs directory exists -# log_dir = "logs" -# if not os.path.exists(log_dir): -# os.makedirs(log_dir) - -# # Configure the file handler -# file_handler = logging.FileHandler("logs/app.log") -# file_handler.setFormatter(formatter) -# file_handler.setLevel(logging.WARNING) -# logger.addHandler(file_handler) - - -class WorkerClient: - def __init__(self, worker_ID,worker_secret,server_URL): - self.worker_id = worker_ID - self.worker_secret = worker_secret - self.server_url = server_URL - if DEBUG_SOCKETIO: - self.sio = socketio.Client(logger=logger, engineio_logger=logger) - else: - self.sio = socketio.Client(logger=False, engineio_logger=False) - self.joined_server = False - - # Initialize Docker client - self.docker_client = docker.from_env() - - # Bind events - self.sio.on("connect", self.on_connect) - self.sio.on("connect_error", self.on_connect_error) - self.sio.on("disconnect", self.on_disconnect) - self.sio.on("task", self.handle_task) - self.sio.on("join_accept", self.on_join_accept) - self.sio.on("join_reject", self.on_join_reject) - self.sio.on("message", self.on_message) - self.sio.on("*", self.debug_all_events) - - def on_connect(self): - logger.info("Connected to the server, requesting to join.") - self.send_join_request() - - - def on_connect_error(self, data): - logger.error(f"Connection failed: {data}") - self.joined_server = False - - def on_disconnect(self): - logger.info("Disconnected from the server.") - self.joined_server = False - - def on_message(self,data): - logger.info(f"Message recieved {data}") - - def on_join_accept(self,data): - logger.info("Join accepted.") - self.joined_server = True - # Start Docker event watcher - asyncio.create_task(self.watch_docker_events()) - - def on_join_reject(self,data): - logger.error("Join rejected.") - self.joined_server = False - exit() - - def send_join_request(self): - """Notify the server about this worker (Initial join).""" - logger.info("Sending join request") - if not self.joined_server: - self.sio.emit("join_request", {"worker_id": self.worker_id,"worker_secret": self.worker_secret}) - logger.info(f"Worker {self.worker_id} asked to join the server.") - - def handle_task(self, data): - """Handle incoming tasks from the server.""" - if not self.joined_server: - logger.error("Not joined to server. ignoring task") - return - task_type = data["type"] - task_id = data["task_id"] - job_details = json.loads(data["job_details"]) - task_worker_id = data["worker_id"] - - logger.info(f"Received task {task_id} of type '{task_type}' with job_details: {job_details}") - job_details_type=type(job_details) - logger.info(f"opbject type {job_details_type}") - - try: - if task_type == "report": - result = ReportTask("",logger).Execute() - elif task_type == "ping": - result = PingTask(job_details,logger).Execute() - elif task_type == "file_presence": - result = FilePresenceTask(job_details,logger).Execute() - elif task_type == "container-create": - result = ContainerTask(job_details,logger).Create() - elif task_type == "container-delete": - result = ContainerTask(job_details,logger).Delete() - else: - raise ValueError(f"Unknown task type: {task_type}") - - self.send_task_result(task_id, result, task_worker_id) - except Exception as e: - logger.error(f"Error processing task {task_id}: {e}") - self.send_task_result( task_id, - {"success": False, "response":str(e)}, - task_worker_id) - - def send_task_result(self, task_id, result, worker_id): - """Send task result back to the server.""" - self.sio.emit("ack", {"task_id": task_id, "worker_id": worker_id, "result": result}) - logger.info(f"Sent result for task {task_id}: {result}") - - def debug_all_events(self, event, data): - """Debug all incoming data.""" - logger.debug(f"Event: {event} | Data: {json.dumps(data, indent=2)}") - - def start(self): - """Worker connects to the API server and processes tasks.""" - try: - logger.info(f"Worker {self.worker_id} connecting to {self.server_url}...") - self.sio.connect(self.server_url) - self.sio.wait() - except Exception as e: - logger.error(f"Error: {e}") - self.sio.disconnect() - - async def watch_docker_events(self): - """Watch Docker events and send them to the server for containers managed by this worker.""" - logger.info("Starting Docker event watcher...") - try: - for event in self.docker_client.events(decode=True): - logger.debug(f"Caught docker event {event}") - if event['Type'] == 'container' and event['Action'] in ['start', 'stop']: - container_id = event['id'] - try: - # Fetch the container details to check its labels - container = self.docker_client.containers.get(container_id) - labels = container.attrs['Config']['Labels'] - - # Check if the container is managed by this worker - if labels and labels.get("managed_by") == "worker_agent": - logger.info(f"Detected Docker event for managed container: {event}") - await self.send_event_to_server(event) - except docker.errors.NotFound: - logger.warning(f"Container {container_id} not found. Skipping event.") - except Exception as e: - logger.error(f"Error fetching container details: {e}") - except Exception as e: - logger.error(f"Error watching Docker events: {e}") - - async def send_event_to_server(self, event): - """Send Docker event to the server over WebSocket.""" - try: - if self.joined_server: - self.sio.emit("docker_event", {"worker_id": self.worker_id, "event": event}) - logger.info(f"Sent Docker event to server: {event}") - except Exception as e: - logger.error(f"Failed to send Docker event to server: {e}") - - -def validate_json_input(text_input): - try: - # Step 1: Validate that it's valid JSON - data = json.loads(text_input) - except json.JSONDecodeError as e: - raise ValueError(f"Invalid JSON format: {e}") - - # Step 2: Ensure 'tenancyID' is present - if 'tenancyID' not in data: - raise ValueError("Key 'tenancyID' is missing.") - - # Step 3: Ensure 1 or more containers are present - if 'containers' not in data or not isinstance(data['containers'], list) or len(data['containers']) == 0: - raise ValueError("Key 'containers' is missing, not a list, or empty.") - - # Step 4: For each container, ensure 'container_name' and 'docker_image' are present - for container in data['containers']: - if not isinstance(container, dict): - raise ValueError("Container is not a dictionary.") - if 'container_name' not in container or 'docker_image' not in container: - raise ValueError("Container is missing 'container_name' or 'docker_image'.") - - # If all validations pass, return the parsed JSON object - return data - - - -def container_test(): - logger.info("Running container test") - payload_text = '''{ - "tenancyID": "tenant123a", - "containers": [ - { - "container_id": "af47632e-43b7-4474-82fa-475a24fe88d5", - "docker_image": "nginx", - "cpu_shares": 1, - "mem_limit": 128, - "container_name": "web-server1", - "command": ["nginx", "-g", "daemon off;"], - "working_dir": "/usr/share/nginx/html" - },{ - "container_id": "82c1c6ae-9dd4-41c0-ba05-45a086d58cf5", - "docker_image": "python:3.9-alpine", - "container_name": "python-webserver", - "cpu_shares": 1, - "mem_limit": 128, - "command": ["python", "-m", "http.server","82"], - "environment": { - "DELAY_START_MSEC": "2000" - }, - "restart_policy": { - "Name": "always" - } - } - ] - - }''' - - # Parse payload - payload_json = validate_json_input(payload_text) - logger.debug(f"Container test payload {payload_json}") - # Initialize and execute ContainerTask - container_task = ContainerTask(payload_json, logger) - result = container_task.Execute() - logger.info(result) - - -import os -import json -import requests -from datetime import datetime - -def enroll_worker(region_id, enrollment_key): - # Configuration - API_URL = "http://127.0.0.1:5000/api/workload_hosts/enroll" - - # Gather system information - def get_system_info(): - try: - with open('/sys/class/dmi/id/sys_vendor', 'r') as f: - system_manufacturer = f.read().strip() - except: - system_manufacturer = "Unknown" - - try: - with open('/sys/class/dmi/id/product_name', 'r') as f: - system_model = f.read().strip() - except: - system_model = "Unknown" - - try: - with open('/sys/class/dmi/id/product_serial', 'r') as f: - physical_identifier = f.read().strip() - except: - physical_identifier = "Unknown" - - try: - dcim_identifier = os.popen('dmidecode -s system-uuid').read().strip() - except: - dcim_identifier = "Unknown" - - installed_date = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - hostname = os.uname().nodename - - return { - "system_manufacturer": system_manufacturer, - "system_model": system_model, - "physical_identifier": physical_identifier, - "dcim_identifier": dcim_identifier, - "installed_date": installed_date, - "hostname": hostname - } - - # Construct JSON payload - system_info = get_system_info() - payload = { - "region_id": region_id, - "region_enrollment_key": enrollment_key, - "system_manufacturer": system_info["system_manufacturer"], - "system_model": system_info["system_model"], - "physical_identifier": system_info["physical_identifier"], - "dcim_identifier": system_info["dcim_identifier"], - "installed_date": system_info["installed_date"], - "hostname": system_info["hostname"] - } - - # Send enrollment request - response = requests.post(API_URL, json=payload, headers={"Content-Type": "application/json"}) - - # Handle response - if response.status_code != 201: - print(f"Enrollment failed: {response.status_code} - {response.text}") - exit(1) - - # Parse response JSON - try: - response_data = response.json() - worker_id = response_data.get("worker_id") - worker_secret = response_data.get("worker_secret") - - if not worker_id or not worker_secret: - print("Enrollment failed: Worker_ID or Worker_Secret not found in response") - exit(1) - - # Ensure .env file exists before appending - if not os.path.exists(".env"): - open(".env", "w").close() # Create an empty .env file if it doesn't exist - - # Append worker credentials to .env - with open(".env", "a") as env_file: - env_file.write(f"WORKER_ID={worker_id}\n") - env_file.write(f"WORKER_SECRET={worker_secret}\n") - - - logger.info("Enrollment successful. Worker_ID and Worker_Secret written to .env file.") - return response_data - - except json.JSONDecodeError: - print("Enrollment failed: Invalid JSON response from server") - exit(1) - -if __name__ == "__main__": - load_dotenv() - - parser = argparse.ArgumentParser(description="Worker CLI") - parser.add_argument("--server-url", help="WebSocket Server URL (e.g., http://localhost:5000)", default=os.getenv("SERVER_URL")) - parser.add_argument("--region-id", help="Region ID(Only required during enrollment)", default=os.getenv("REGION_ID")) - parser.add_argument("--region-enrollment-key", help="Region Enrollment Key", default=os.getenv("REGION_ENROLLMENT_KEY")) - parser.add_argument("--container-test", action="store_true", help="Run container test") - args = parser.parse_args() - - # Verbose logging of environment variables - logger.info("Starting Worker CLI") - logger.info(f"SERVER_URL: {args.server_url if args.server_url else 'Not Provided'}") - logger.info(f"REGION_ID: {args.region_id if args.region_id else 'Not Provided'}") - logger.info(f"REGION_ENROLLMENT_KEY: {args.region_enrollment_key if args.region_enrollment_key else 'Not Provided'}") - - if args.container_test: - logger.info("Running container test...") - container_test() - exit() - - # Ensure required arguments are provided - if not args.server_url: - parser.error("--server-url are required when --container-test is not used and not defined in .env.") - - - - # Enroll worker only if WORKER_ID and WORKER_SECRET are not in .env - worker_secret = os.getenv("WORKER_SECRET") - worker_id = os.getenv("WORKER_ID") - if not worker_id or not worker_secret: - logger.info("Worker ID or Worker Secret missing. Enrolling worker...") - #Region ID is only requried for enrollment - if not args.region_id: - parser.error("--region-id must be provided either in the .env file or via CLI.") - if not args.region_enrollment_key: - parser.error("--region-enrollment-key must be provided either in the .env file or via CLI.") - - enroll_worker(args.region_id,args.region_enrollment_key) - load_dotenv() - else: - logger.info("Worker ID and Worker Secret found. Skipping enrollment.") - - - worker_ID=os.getenv("WORKER_ID") - worker_secret=os.getenv("WORKER_SECRET") - server_URL=os.getenv("SERVER_URL") if os.getenv("SERVER_URL") else args.server_url - - # Create and start the worker - logger.info("Starting Worker with the folowing params...") - logger.info(f"WORKER_ID: {worker_ID}") - logger.info(f"WORKER_SECRET: {worker_secret}") - logger.info(f"SERVER_URL: {server_URL}") - worker = WorkerClient(worker_ID,worker_secret,server_URL) - worker.start() diff --git a/worker/__init__.py b/worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/worker/docker.py b/worker/docker.py new file mode 100644 index 0000000..f333257 --- /dev/null +++ b/worker/docker.py @@ -0,0 +1,59 @@ +import docker +from logger import logger + +class DockerMonitor: + def __init__(self, event_queue): + self.event_queue = event_queue + self.docker_client = docker.from_env() + self.running = True + + def run(self): + """Monitor Docker events and extract the 'managed_by' tag if it exists.""" + logger.info("Docker monitor started") + + # Get events from Docker + events = self.docker_client.events(decode=True) + + for event in events: + if not self.running: + break + + if event.get('Type') == 'container': + # Process container events (start, stop, die, etc.) + status = event.get('status') + container_id = event.get('id') + container_name = "" + + if status in ['start', 'stop', 'die', 'create', 'destroy']: + # Extract the 'managed_by' tag from the container labels (if it exists) + managed_by = None + container_attributes = event.get('Actor', {}).get('Attributes', {}) + if container_attributes: + managed_by = container_attributes.get('managed_by') + container_name = container_attributes.get('name') + system_container_id = container_attributes.get('system_container_id') + + if not managed_by=="worker_agent": + logger.debug(f"Ignoring {container_id}, not managed by us") + continue + + # Prepare the event data + event_data = { + 'source': 'docker', + 'event_type': status, + 'container_id': container_id, + 'system_container_id': system_container_id, + 'container_name': container_name, + 'timestamp': event.get('time'), + 'details': event, + 'managed_by': managed_by # Add the 'managed_by' tag to the payload + } + + # Put event in the queue for the worker client + self.event_queue.put(event_data) + logger.info(f"Added Docker event to queue: {status} for container {container_id}. Managed by: {managed_by}") + + def stop(self): + """Stop the Docker monitor""" + self.running = False + self.docker_client.close() \ No newline at end of file diff --git a/worker/libvirt.py b/worker/libvirt.py new file mode 100644 index 0000000..b0e48fd --- /dev/null +++ b/worker/libvirt.py @@ -0,0 +1,215 @@ +from logger import logger +import libvirt +import time +import xml.etree.ElementTree as ET +import threading + +class LibvirtMonitor: + def __init__(self, event_queue): + self.event_queue = event_queue + self.conn = None + self.running = True + self.event_impl = None + + def connect_libvirt(self): + """Connect to libvirt""" + try: + # Try to connect to local libvirt + self.conn = libvirt.open('qemu:///system') + + # Initialize the event implementation + libvirt.virEventRegisterDefaultImpl() + self.event_impl = True + + return True + except libvirt.libvirtError as e: + logger.error(f"Failed to connect to libvirt: {e}") + return False + + def callback_lifecycle(self, conn, dom, event, detail, opaque): + """Callback for domain lifecycle events""" + event_names = { + libvirt.VIR_DOMAIN_EVENT_STARTED: 'started', + libvirt.VIR_DOMAIN_EVENT_STOPPED: 'stopped', + libvirt.VIR_DOMAIN_EVENT_SUSPENDED: 'suspended', + libvirt.VIR_DOMAIN_EVENT_RESUMED: 'resumed', + libvirt.VIR_DOMAIN_EVENT_SHUTDOWN: 'shutdown', + libvirt.VIR_DOMAIN_EVENT_DEFINED: 'defined', + libvirt.VIR_DOMAIN_EVENT_UNDEFINED: 'undefined' + } + + event_name = event_names.get(event, f'unknown-{event}') + + # Get domain information + try: + domain_name = dom.name() + domain_id = dom.ID() + + # Check metadata for 'vm_id' + vm_id = self.get_vm_id_from_metadata(dom) + if vm_id is None: + logger.warning("NO VMID found, likely not a VM managed by us, ignoring") + return 0 + logger.debug(f"Found VM_ID {vm_id}") + + event_data = { + 'source': 'libvirt', + 'event_type': event_name, + 'system_vm_id': vm_id, + 'libvirt_vm_id': domain_id, + 'timestamp': time.time() + } + + + # Get the VNC port if available + vnc_port = self.get_vnc_port(dom) + if vnc_port is not None: + event_data['vnc_port'] = vnc_port + + + + + # Put event in the queue for the worker client + self.event_queue.put(event_data) + logger.info(f"Added libvirt event to queue: {event_name} for domain {domain_name}") + + except libvirt.libvirtError as e: + logger.error(f"Error getting domain info: {e}") + + def get_vnc_port(self, dom): + """Extract the VNC port from the domain XML""" + try: + xml_desc = dom.XMLDesc(0) + root = ET.fromstring(xml_desc) + + # Find the graphics element with type 'vnc' + graphics = root.find(".//graphics[@type='vnc']") + if graphics is not None: + port = graphics.get('port') + if port is not None: + return int(port) + except Exception as e: + logger.error(f"Error extracting VNC port: {e}") + return None + + def get_vm_id_from_metadata(self, dom): + """ + Extract the VM ID from the XML configuration. + + Args: + xml_config (str): XML configuration of the VM. + logger: Logger instance for logging. + + Returns: + str: The VM ID if found, otherwise None. + """ + try: + # Parse the XML configuration + xml_desc = dom.XMLDesc(0) + root = ET.fromstring(xml_desc) + + # Define the custom namespace + namespaces = { + "custom": "http://example.com/xmlns/libvirt/custom" + } + + # Find the metadata section + metadata = root.find(".//metadata") + + if metadata is None: + logger.warning("Metadata section not found in the XML configuration.") + return None + + logger.debug(f"Metadata section: {ET.tostring(metadata)}") + + # Look for the 'vm_id' tag within the custom namespace + virtual_machine_id_tag = metadata.find(".//custom:virtual_machine_id", namespaces) + logger.debug(f"VM ID tag: {virtual_machine_id_tag}") + + if virtual_machine_id_tag is not None: + return virtual_machine_id_tag.text + + logger.warning("VM ID not found in the metadata section.") + return None + + except Exception as e: + logger.error(f"Error extracting VM ID from XML: {e}") + raise + + + + def run(self): + """Monitor libvirt events""" + logger.info("Libvirt monitor started") + + try: + if not self.connect_libvirt(): + logger.error("Failed to start libvirt monitor") + return + + # Register for domain lifecycle events + cb_id = self.conn.domainEventRegisterAny( + None, + libvirt.VIR_DOMAIN_EVENT_ID_LIFECYCLE, + self.callback_lifecycle, + None + ) + + + def event_loop_thread(): + while self.running: + try: + libvirt.virEventRunDefaultImpl() + time.sleep(0.1) # Small delay to prevent CPU thrashing + except Exception as e: + logger.error(f"Error in libvirt event loop: {e}") + time.sleep(1) # Longer delay after error + + # Start event loop thread + event_thread = threading.Thread(target=event_loop_thread) + event_thread.daemon = True + event_thread.start() + + # Main monitor loop - periodically check domains + while self.running: + try: + # Poll for active domains as a fallback mechanism + domains = self.conn.listAllDomains() + for domain in domains: + state, reason = domain.state() + domain_name = domain.name() + # Just log the current state, don't generate events + # This is just to keep the connection active + # logger.debug(f"Domain {domain_name} state: {state}") + except libvirt.libvirtError as e: + logger.error(f"Error polling libvirt domains: {e}") + # Try to reconnect + try: + if self.conn: + self.conn.close() + self.connect_libvirt() + except Exception as e: + logger.error(f"Failed to reconnect to libvirt: {e}") + + time.sleep(5) # Check every 5 seconds + + # Cleanup + try: + if cb_id >= 0: + self.conn.domainEventDeregisterAny(cb_id) + except Exception: + pass + + except Exception as e: + logger.error(f"Fatal error in libvirt monitor: {e}") + finally: + # Close connection + if self.conn: + try: + self.conn.close() + except Exception: + pass + + def stop(self): + """Stop the libvirt monitor""" + self.running = False \ No newline at end of file diff --git a/worker/worker.py b/worker/worker.py new file mode 100644 index 0000000..eb11924 --- /dev/null +++ b/worker/worker.py @@ -0,0 +1,193 @@ +import socketio +from logger import logger +import json +import asyncio +from worker_tasks.container import ContainerTask +from worker_tasks.file_presence import FilePresenceTask +from worker_tasks.libvirt import LibvirtVirtualMachineTask +from worker_tasks.ping import PingTask +from worker_tasks.report import ReportTask + +class WorkerClient: + def __init__(self, worker_ID, worker_secret, server_URL, event_queue=None,DEBUG_SOCKETIO=False): + self.worker_id = worker_ID + self.worker_secret = worker_secret + self.server_url = server_URL + self.event_queue = event_queue + self.running = True + self.libvirt_config={ + "default_volume_path": "/tmp" + } + + if DEBUG_SOCKETIO: + self.sio = socketio.AsyncClient(logger=logger, engineio_logger=logger) + else: + self.sio = socketio.AsyncClient(logger=False, engineio_logger=False) + self.joined_server = False + + # Bind events + self.sio.on("connect", self.on_connect) + self.sio.on("connect_error", self.on_connect_error) + self.sio.on("disconnect", self.on_disconnect) + self.sio.on("task", self.handle_task) + self.sio.on("join_accept", self.on_join_accept) + self.sio.on("join_reject", self.on_join_reject) + self.sio.on("message", self.on_message) + self.sio.on("*", self.debug_all_events) + + async def on_connect(self): + logger.info("Connected to the server, requesting to join.") + await self.send_join_request() + + async def on_connect_error(self, data): + logger.error(f"Connection failed: {data}") + self.joined_server = False + + async def on_disconnect(self): + logger.info("Disconnected from the server.") + self.joined_server = False + + async def on_message(self, data): + logger.info(f"Message received: {data}") + + async def on_join_accept(self, data): + logger.info("Join accepted.") + self.joined_server = True + + async def on_join_reject(self, data): + logger.error("Join rejected.") + self.joined_server = False + exit() + + async def send_join_request(self): + """Notify the server about this worker (Initial join).""" + logger.info("Sending join request") + if not self.joined_server: + await self.sio.emit("join_request", {"worker_id": self.worker_id, "worker_secret": self.worker_secret}) + logger.info(f"Worker {self.worker_id} asked to join the server.") + + async def handle_task(self, data): + """Handle incoming tasks from the server.""" + if not self.joined_server: + logger.error("Not joined to server. ignoring task") + return + task_type = data["type"] + task_id = data["task_id"] + job_details = json.loads(data["job_details"]) + task_worker_id = data["worker_id"] + + logger.info(f"Received task {task_id} of type '{task_type}' with job_details: {job_details}") + + try: + if task_type == "report": + result = ReportTask("", logger).Execute() + elif task_type == "ping": + result = PingTask(job_details, logger).Execute() + elif task_type == "file_presence": + result = FilePresenceTask(job_details, logger).Execute() + elif task_type == "container-create": + result = ContainerTask(job_details, logger).Create() + elif task_type == "container-delete": + result = ContainerTask(job_details, logger).Delete() + elif task_type == "virtual-machine-create": + result = LibvirtVirtualMachineTask(job_details,self.libvirt_config, logger).execute() + elif task_type == "virtual-machine-delete": + result = LibvirtVirtualMachineTask(job_details,self.libvirt_config, logger).execute() + else: + raise ValueError(f"Unknown task type: {task_type}") + + + await self.send_task_result(task_id, result, task_worker_id) + except Exception as e: + logger.error(f"Error processing task {task_id}: {e}") + await self.send_task_result(task_id, {"success": False, "response": str(e)}, task_worker_id) + + async def send_task_result(self, task_id, result, worker_id): + """Send task result back to the server.""" + await self.sio.emit("ack", {"task_id": task_id, "worker_id": worker_id, "result": result}) + logger.info(f"Sent result for task {task_id}: {result}") + + async def send_event(self, event_data): + """Send monitoring event to the server, handling libvirt and docker events separately.""" + if not self.joined_server: + return + + source = event_data.get('source', 'unknown') + event_type = event_data.get('event_type', 'event') + + # Construct the base event payload + payload = { + "worker_id": self.worker_id, + "type": f"{source}_{event_type}", + "details": event_data + } + + # Handle libvirt events + if source == "libvirt": + payload["details"] = { + "system_vm_id": event_data.get("system_vm_id"), + "libvirt_vm_id": event_data.get("libvirt_vm_id"), + "timestamp": event_data.get("timestamp"), + } + + # Emit the event to the server + await self.sio.emit("libvirt_event", payload) + logger.info(f"Sent libvirt event: {event_type}\n{payload}") + + + # Handle docker events + elif source == "docker": + payload["details"] = { + "container_id": event_data.get("container_id"), + "system_container_id": event_data.get("system_container_id"), + "container_name": event_data.get("container_name"), + "timestamp": event_data.get("timestamp"), + "status": event_data.get("details", {}).get("status"), + "image": event_data.get("details", {}).get("from"), + "action": event_data.get("details", {}).get("Action") + } + + # Emit the event to the server + await self.sio.emit("docker_event", payload) + logger.info(f"Sent {source} docker event: {event_type}\n{payload}") + + async def debug_all_events(self, event, data): + """Debug all incoming data.""" + logger.debug(f"Event: {event} | Data: {json.dumps(data, indent=2)}") + + async def process_event_queue(self): + """Process events from the queue and send them to the server.""" + if not self.event_queue: + return + + while self.running: + try: + # Non-blocking check for events + if not self.event_queue.empty(): + event = self.event_queue.get(block=False) + logger.info(f"Processing event from queue: {event}") + await self.send_event(event) + await asyncio.sleep(0.1) # Small delay to prevent CPU thrashing + except Exception as e: + logger.error(f"Error in event processing: {e}") + + async def start(self): + """Worker connects to the API server and processes tasks.""" + try: + logger.info(f"Worker {self.worker_id} connecting to {self.server_url}...") + await self.sio.connect(self.server_url) + + # Start the event queue processor if we have an event queue + if self.event_queue: + event_processor = asyncio.create_task(self.process_event_queue()) + await self.sio.wait() + event_processor.cancel() + else: + await self.sio.wait() + except Exception as e: + logger.error(f"Error: {e}") + await self.sio.disconnect() + + def stop(self): + """Stop the worker client.""" + self.running = False diff --git a/worker2.py b/worker2.py new file mode 100644 index 0000000..6b57549 --- /dev/null +++ b/worker2.py @@ -0,0 +1,280 @@ +import multiprocessing as mp +from multiprocessing import Queue +import json +import signal +import sys +import asyncio +import docker +import os +import argparse +import requests +from datetime import datetime +from dotenv import load_dotenv +from logger import logger + +from worker.docker import DockerMonitor +from worker.libvirt import LibvirtMonitor +from worker.worker import WorkerClient + +# Debug flag +DEBUG_SOCKETIO = False + + +def enroll_worker(region_id, api_base_url="http://127.0.0.1:5000"): + """Enroll the worker with the server and get worker credentials""" + API_URL = f"{api_base_url}/api/workload_hosts/enroll" + + # Gather system information + def get_system_info(): + try: + with open('/sys/class/dmi/id/sys_vendor', 'r') as f: + system_manufacturer = f.read().strip() + except: + system_manufacturer = "Unknown" + + try: + with open('/sys/class/dmi/id/product_name', 'r') as f: + system_model = f.read().strip() + except: + system_model = "Unknown" + + try: + with open('/sys/class/dmi/id/product_serial', 'r') as f: + physical_identifier = f.read().strip() + except: + physical_identifier = "Unknown" + + try: + dcim_identifier = os.popen('dmidecode -s system-uuid').read().strip() + except: + dcim_identifier = "Unknown" + + installed_date = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + hostname = os.uname().nodename + + return { + "system_manufacturer": system_manufacturer, + "system_model": system_model, + "physical_identifier": physical_identifier, + "dcim_identifier": dcim_identifier, + "installed_date": installed_date, + "hostname": hostname + } + + # Construct JSON payload + system_info = get_system_info() + payload = { + "region_id": region_id, + "system_manufacturer": system_info["system_manufacturer"], + "system_model": system_info["system_model"], + "physical_identifier": system_info["physical_identifier"], + "dcim_identifier": system_info["dcim_identifier"], + "installed_date": system_info["installed_date"], + "hostname": system_info["hostname"] + } + + # Send enrollment request + logger.info(f"Sending enrollment request to {API_URL}") + response = requests.post(API_URL, json=payload, headers={"Content-Type": "application/json"}) + + # Handle response + if response.status_code != 201: + logger.error(f"Enrollment failed: {response.status_code} - {response.text}") + exit(1) + + # Parse response JSON + try: + response_data = response.json() + worker_id = response_data.get("worker_id") + worker_secret = response_data.get("worker_secret") + + if not worker_id or not worker_secret: + logger.error("Enrollment failed: Worker_ID or Worker_Secret not found in response") + exit(1) + + # Ensure .env file exists before appending + if not os.path.exists(".env"): + open(".env", "w").close() # Create an empty .env file if it doesn't exist + + # Append worker credentials to .env + with open(".env", "a") as env_file: + env_file.write(f"WORKER_ID={worker_id}\n") + env_file.write(f"WORKER_SECRET={worker_secret}\n") + + logger.info("Enrollment successful. Worker_ID and Worker_Secret written to .env file.") + return response_data + + except json.JSONDecodeError: + logger.error("Enrollment failed: Invalid JSON response from server") + exit(1) + + +def container_test(): + """Run a simple test to verify Docker functionality""" + try: + client = docker.from_env() + containers = client.containers.list() + logger.info(f"Container test successful. Found {len(containers)} running containers.") + for container in containers: + logger.info(f"Container: {container.name} (ID: {container.short_id})") + return True + except Exception as e: + logger.error(f"Container test failed: {e}") + return False + + +def libvirt_test(): + """Run a simple test to verify libvirt functionality""" + try: + conn = libvirt.open('qemu:///system') + domains = conn.listAllDomains() + logger.info(f"Libvirt test successful. Found {len(domains)} domains.") + for domain in domains: + logger.info(f"Domain: {domain.name()} (ID: {domain.ID()})") + conn.close() + return True + except Exception as e: + logger.error(f"Libvirt test failed: {e}") + return False + + +# Worker process function +def worker_process_func(event_queue, worker_id, worker_secret, server_url): + """Function to run the worker client in a separate process""" + worker = WorkerClient(worker_id, worker_secret, server_url, event_queue,DEBUG_SOCKETIO) + + # Run the asyncio event loop + async def main(): + await worker.start() + + try: + asyncio.run(main()) + except KeyboardInterrupt: + pass + except Exception as e: + logger.error(f"Worker process error: {e}") + finally: + worker.stop() + + +def main(): + # Load environment variables + load_dotenv() + + # Parse command line arguments + parser = argparse.ArgumentParser(description="Worker CLI with Docker and Libvirt monitoring") + parser.add_argument("--server-url", help="WebSocket Server URL (e.g., http://localhost:5000)", default=os.getenv("SERVER_URL")) + parser.add_argument("--region-id", help="Region ID (Only required during enrollment)", default=os.getenv("REGION_ID")) + parser.add_argument("--container-test", action="store_true", help="Run container test and exit") + parser.add_argument("--libvirt-test", action="store_true", help="Run libvirt test and exit") + parser.add_argument("--no-docker", action="store_true", help="Disable Docker monitoring") + parser.add_argument("--no-libvirt", action="store_true", help="Disable Libvirt monitoring") + parser.add_argument("--api-base-url", help="Base URL for API (used for enrollment)", default=os.getenv("API_BASE_URL", "http://127.0.0.1:5000")) + + args = parser.parse_args() + args = parser.parse_args() + + # Verbose logging of environment variables + logger.info("Starting Worker CLI with monitoring capabilities") + logger.info(f"SERVER_URL: {args.server_url if args.server_url else 'Not Provided'}") + logger.info(f"REGION_ID: {args.region_id if args.region_id else 'Not Provided'}") + logger.info(f"API_BASE_URL: {args.api_base_url}") + + # Run tests if requested + if args.container_test: + logger.info("Running container test...") + container_test() + exit() + + if args.libvirt_test: + logger.info("Running libvirt test...") + libvirt_test() + exit() + + # Ensure required arguments are provided + if not args.server_url: + parser.error("--server-url is required when not defined in .env.") + + # Enroll worker only if WORKER_ID and WORKER_SECRET are not in .env + worker_secret = os.getenv("WORKER_SECRET") + worker_id = os.getenv("WORKER_ID") + if not worker_id or not worker_secret: + logger.info("Worker ID or Worker Secret missing. Enrolling worker...") + # Region ID is only required for enrollment + if not args.region_id: + parser.error("--region-id must be provided either in the .env file or via CLI.") + + enroll_worker(args.region_id, args.api_base_url) + # Reload environment variables to get the new worker credentials + load_dotenv() + else: + logger.info("Worker ID and Worker Secret found. Skipping enrollment.") + + # Get worker credentials + worker_id = os.getenv("WORKER_ID") + worker_secret = os.getenv("WORKER_SECRET") + server_url = os.getenv("SERVER_URL") if os.getenv("SERVER_URL") else args.server_url + + # Create a queue for inter-process communication + event_queue = Queue() + + # Create monitoring processes + processes = [] + monitors = [] + + # Create worker process + logger.info("Starting Worker with the following params...") + logger.info(f"WORKER_ID: {worker_id}") + logger.info(f"WORKER_SECRET: {worker_secret}") + logger.info(f"SERVER_URL: {server_url}") + worker_proc = mp.Process(target=worker_process_func, args=(event_queue, worker_id, worker_secret, server_url)) + processes.append(worker_proc) + + # Create Docker monitor process if enabled + if not args.no_docker: + logger.info("Docker monitoring enabled") + docker_monitor = DockerMonitor(event_queue) + docker_proc = mp.Process(target=docker_monitor.run) + processes.append(docker_proc) + monitors.append(docker_monitor) + else: + logger.warning("Docker monitoring disabled") + + # Create Libvirt monitor process if enabled + if not args.no_libvirt: + logger.info("Libvirt monitoring enabled") + libvirt_monitor = LibvirtMonitor(event_queue) + libvirt_proc = mp.Process(target=libvirt_monitor.run) + processes.append(libvirt_proc) + monitors.append(libvirt_monitor) + else: + logger.warning("Libvirt monitoring disabled") + + # Start all processes + for process in processes: + process.start() + + # Handle graceful shutdown + def signal_handler(sig, frame): + logger.info("Shutting down...") + for monitor in monitors: + monitor.stop() + + for process in processes: + process.join(timeout=5) + if process.is_alive(): + process.terminate() + + sys.exit(0) + + # Register signal handlers + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Wait for processes to finish + for process in processes: + process.join() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/worker_tasks/container.py b/worker_tasks/container.py index 1ad07b7..6a4e382 100644 --- a/worker_tasks/container.py +++ b/worker_tasks/container.py @@ -1,6 +1,5 @@ import docker import json -import logging from logging import Logger @@ -15,8 +14,7 @@ class ContainerTask: self.params = params self.logger = logger - container_count = len(params['containers']) - logger.info(f"ContainerTask initialized successfully, preparing to process {container_count} containers") + logger.info(f"ContainerTask initialized successfully") @@ -84,21 +82,6 @@ class ContainerTask: return None # Master container not found - - - - def check_or_create_nscontroller(self, client, tenancyID): - """Check if a namespace controller exists for the given tenancy ID. If not, create one.""" - nscontroller_name = f"nscontroller-{tenancyID}" - self.logger.info(f"Checking for namespace controller '{nscontroller_name}'...") - containers = client.containers.list(all=True, filters={"name": nscontroller_name}) - if not containers: - self.logger.info(f"Namespace controller '{nscontroller_name}' does not exist. Creating...") - client.containers.run("busybox", name=nscontroller_name, detach=True, command="sleep infinite") - self.logger.info(f"Namespace controller '{nscontroller_name}' created successfully.") - else: - self.logger.info(f"Namespace controller '{nscontroller_name}' already exists.") - return nscontroller_name def delete_container(self, client, container_name): """Delete the specified container if it exists.""" @@ -113,12 +96,12 @@ class ContainerTask: else: self.logger.info(f"Container '{container_name}' does not exist. No action needed.") - def launch_container(self, client, container, nscontroller_name): + def launch_container(self, client, container): """Launch the container as specified in the container variable, attaching it to the namespace controller.""" container_name = container['container_id'] # Check if the container is marked for deletion - if container.get('deleted', False): + if container.get('desired_state', False): self.logger.info(f"Container '{container_name}' is marked for deletion. Ensuring it is not present...") self.delete_container(client, container_name) return # Exit the function as no further action is needed for deleted containers @@ -167,15 +150,37 @@ class ContainerTask: self.logger.info(f"Removing container '{container_name}' and recreating with the new configuration...") self.delete_container(client, container_name) + #Check if the NSController container exists - self.logger.info(f"Launching container '{container_name}'...") - # Add the logic to launch the container here + + nscontroller_container_name=container['NSController_launchparams']['container_name'] + self.logger.debug(f"Name of NSController is {nscontroller_container_name}") + running_NSController = client.containers.list(all=True, filters={"name": nscontroller_container_name}) + self.logger.debug(f"{running_NSController}") + if not running_NSController: + self.logger.info("NSController does not exist, creating it") + nscontroller_container_config = { + "image": container['NSController_launchparams']['docker_image'], + "name": container['NSController_launchparams']['container_name'], + "command": container['NSController_launchparams']['command'], + "network": "none", + "detach": True, + "labels": { + "managed_by": "worker_agent", + "system_container_id": container['NSController_launchparams']['container_id'] + } + } + # Launch the container + client.containers.run(**nscontroller_container_config) + self.logger.info(f"Container '{nscontroller_container_config}' launched successfully.") + + self.logger.info(f"Launching container '{container_name}'...") # Prepare container configuration container_config = { "image": container['docker_image'], "name": container_name, - "network": f"container:{nscontroller_name}", + "network": f"container:{nscontroller_container_name}", "detach": True, "labels": {"managed_by": "worker_agent"} # Add a label to identify containers started by this worker } @@ -211,16 +216,15 @@ class ContainerTask: client = docker.from_env() self.logger.info("Docker client initialized.") - # Check or create namespace controller - nscontroller_name = self.check_or_create_nscontroller(client, self.params['tenancyID']) - # Launch the container - all_containers= json.loads(self.params['containers']) + _container= json.loads(self.params['container']) - for _container in all_containers: - self.logger.info(_container) - create_result=self.launch_container(client, _container, nscontroller_name) - #TODO - Track the resultsof the tasks so we can rollback\delete if any subsequent part of the task fails + self.logger.info(_container) + # Is this container a NS controller or regular container? + # container_workload_type=self.params['container'] + + create_result=self.launch_container(client, _container) + #TODO - Track the resultsof the tasks so we can rollback\delete if any subsequent part of the task fails self.logger.info("ContainerTask executed successfully.") @@ -235,7 +239,7 @@ class ContainerTask: def Delete(self): """Delete the specified container and its namespace controller if it's the last one.""" - container_name=self.params['containers'][0]['container_id'] + container_name=self.params['container'][0]['container_id'] self.logger.info(f"Executing ContainerTask - Delete for container '{container_name}'...") try: # Initialize Docker client diff --git a/worker_tasks/libvirt.py b/worker_tasks/libvirt.py index 5903c19..7bb5400 100644 --- a/worker_tasks/libvirt.py +++ b/worker_tasks/libvirt.py @@ -1,31 +1,46 @@ +import re import libvirt import difflib +import json +import os +import subprocess from logging import Logger +import xml.etree.ElementTree as ET +from xml.dom import minidom + +import requests +import hashlib +import gzip +import shutil -class LibvirtVMTask: - def __init__(self, params, logger): +class LibvirtVirtualMachineTask: + def __init__(self, params, config, logger): """ - Initialize the LibvirtVMTask. + Initialize the LibvirtVirtualMachineTask. Args: - params (dict): Contains 'vm_name', 'desired_state', and optional 'action' and 'xml_config'. + params (dict): Contains 'virtual_machine_id', 'desired_state', optional 'action' and 'virtual_machine_config'/'xml_config'. + virtual_machine_config (dict): VirtualMachine configuration details including CPU, RAM, networks, volumes. + xml_config (str): Optional direct XML configuration (used if virtual_machine_config not provided). + config (dict): Configuration including default volume path. logger (Logger): Logger instance for logging. """ if not isinstance(logger, Logger): raise TypeError("Logger must be an instance of the logging.Logger class.") self.logger = logger - self.logger.info("Initializing LibvirtVMTask") + self.logger.info("Initializing LibvirtVirtualMachineTask") # Validate params required_params = { - "vm_name": str, + "virtual_machine_id": str, "desired_state": str, # e.g., 'running', 'stopped', 'deleted' } optional_params = { "action": str, # 'reboot', 'start', 'stop' - "xml_config": str, # Only required for 'create' + "xml_config": str, # Direct XML configuration + "virtual_machine_config": dict, # JSON configuration for VirtualMachine details } if not isinstance(params, dict): @@ -45,15 +60,350 @@ class LibvirtVMTask: self.logger.error(f"Parameter '{key}' must be of type {expected_type.__name__}.") raise TypeError(f"Parameter '{key}' must be of type {expected_type.__name__}.") - self.vm_name = params["vm_name"] + self.virtual_machine_id = params["virtual_machine_id"] self.desired_state = params["desired_state"] self.action = params.get("action") + + # Handle VirtualMachine configuration + self.virtual_machine_config = params.get("virtual_machine_config") self.xml_config = params.get("xml_config") - self.logger.info(f"LibvirtVMTask initialized with vm_name={self.vm_name}, desired_state={self.desired_state}") + self.volume_paths = [] # Will store volume paths for cleaning up + + # Config entity + self.config = config + if not isinstance(self.config, dict) or "default_volume_path" not in self.config: + self.logger.error("Config must be a dictionary containing 'default_volume_path'.") + raise ValueError("Config must be a dictionary containing 'default_volume_path'.") + + # If virtual_machine_config is provided, ensure volumes exist and generate XML from it + if self.virtual_machine_config: + self.logger.info(f"VirtualMachine configuration provided as JSON. Processing volumes and generating XML.") + # Handle storage volumes if needed + if "volumes" in self.virtual_machine_config: + self._process_storage_volumes() + self.xml_config = self._generate_xml_from_config() + elif not self.xml_config and self.desired_state != "deleted": + self.logger.error("Either virtual_machine_config or xml_config must be provided for non-delete operations.") + raise ValueError("Either virtual_machine_config or xml_config must be provided for non-delete operations.") + + self.logger.info(f"LibvirtVirtualMachineTask initialized with virtual_machine_id={self.virtual_machine_id}, desired_state={self.desired_state}") + + def _get_best_machine_type(self): + """ + Determine the best available 'pc-q35' machine type for KVM on x86_64. + Returns the highest version found in libvirt capabilities. + """ + self.logger.info("Determining best machine type from libvirt capabilities.") + + # Connect to libvirt + try: + with libvirt.open("qemu:///system") as conn: + if conn is None: + raise RuntimeError("Failed to connect to libvirt.") + caps_xml = conn.getCapabilities() + except libvirt.libvirtError as e: + self.logger.error(f"Libvirt connection error: {e}") + raise RuntimeError(f"Failed to connect to libvirt: {e}") + + # Parse capabilities XML + try: + root = ET.fromstring(caps_xml) + + # Find x86_64 architecture + arch_elem = root.find(".//guest/arch[@name='x86_64']") + if not arch_elem: + self.logger.error("x86_64 architecture not found in libvirt capabilities.") + raise ValueError("x86_64 architecture not supported by the host.") + + # Check for KVM domain + has_kvm = any(domain.get("type") == "kvm" for domain in arch_elem.findall("domain")) + self.logger.debug(f"KVM domain found: {has_kvm}") + + if not has_kvm: + self.logger.error("KVM domain not found within x86_64 architecture.") + raise ValueError("KVM domain not supported for x86_64.") + + # Find all pc-q35 machine types + q35_machines = [ + machine.text for machine in arch_elem.findall("machine") + if machine.text and machine.text.startswith("pc-q35-") + ] + + if not q35_machines: + self.logger.error("No 'pc-q35' machine types available for KVM.") + raise ValueError("No 'pc-q35' machine types available for KVM.") + + # Use a more robust version comparison with packaging.version + from packaging import version + + # Filter to include only standard versioned machine types (major.minor format) + version_pattern = re.compile(r"^pc-q35-(\d+\.\d+)$") + valid_machines = [m for m in q35_machines if version_pattern.match(m)] + + if not valid_machines: + self.logger.warning("No standard versioned pc-q35 machine types found.") + # Fall back to the original list if no standard versions found + highest_version = max(q35_machines, key=lambda x: x.replace("pc-q35-", "")) + else: + # Sort based on actual version comparisons rather than float conversion + highest_version = max( + valid_machines, + key=lambda x: version.parse(x.replace("pc-q35-", "")) + ) + + self.logger.info(f"Selected machine type: {highest_version}") + return highest_version + + except Exception as e: + self.logger.error(f"Error parsing libvirt capabilities: {e}") + raise RuntimeError(f"Failed to determine best machine type: {e}") + + + + def _process_storage_volumes(self): + """ + Process storage volumes defined in virtual_machine_config. + Create any volumes that don't exist. + Keep track of volume paths for potential cleanup during delete. + """ + self.logger.info("Processing storage volumes") + for volume in self.virtual_machine_config["volumes"]: + if "id" not in volume: + self.logger.error("Volume configuration missing required 'id' attribute.") + raise ValueError("Volume configuration missing required 'id' attribute.") + + volume_id = volume["id"] + format_type = volume.get("format", "qcow2") # Default qcow2 + path = self._get_volume_path(volume_id, format_type) + self.volume_paths.append(path) + + # Check if volume exists, create if it doesn't + if not self._check_volume_exists(path): + # Get optional volume parameters with defaults + size_gb = volume.get("size_gb", 10) # Default 10GB + backing_file = volume.get("backing_file", None) + + self._create_volume(path, size_gb, format_type, backing_file) + + def _get_volume_path(self, volume_id, format_type): + """ + Construct the full path for a volume using the default volume path and volume ID. + + Args: + volume_id (str): The ID of the volume. + format_type (str): The format of the volume (e.g., 'qcow2'). + + Returns: + str: The full path to the volume. + """ + if format_type == "qcow2": + return os.path.join(self.config["default_volume_path"], f"{volume_id}.qcow2") + else: + return os.path.join(self.config["default_volume_path"], volume_id) + + def _check_volume_exists(self, path): + """ + Check if a storage volume exists at the specified path. + + Args: + path (str): Path to the storage volume. + + Returns: + bool: True if volume exists, False otherwise. + """ + self.logger.info(f"Checking if volume exists at path: {path}") + return os.path.exists(path) + + def _create_volume(self, path, size_gb=10, format_type="qcow2", backing_file=None): + """ + Create a new storage volume. + + Args: + path (str): Path to the volume. + size_gb (int): Size of the volume in GB. + format_type (str): Format of the volume (qcow2, raw, etc.) + backing_file (str): Optional path to a backing file for thin provisioning. + + Returns: + bool: True if creation successful, raises exception otherwise. + """ + self.logger.info(f"Creating volume at path: {path} with size: {size_gb}GB, format: {format_type}") + + # Create directory if it doesn't exist + directory = os.path.dirname(path) + if not os.path.exists(directory): + self.logger.info(f"Creating directory: {directory}") + os.makedirs(directory, exist_ok=True) + + # Build qemu-img command + command = ["qemu-img", "create"] + command.extend(["-f", format_type]) + + # Add backing file if specified (for thin provisioning) + if backing_file: + if not os.path.exists(backing_file): + raise ValueError(f"Backing file does not exist: {backing_file}") + command.extend(["-b", backing_file]) + + command.extend([path, f"{size_gb}G"]) + + # Execute command + try: + self.logger.info(f"Executing command: {' '.join(command)}") + result = subprocess.run(command, check=True, capture_output=True, text=True) + self.logger.info(f"Volume created successfully: {path}") + return True + except subprocess.CalledProcessError as e: + self.logger.error(f"Failed to create volume: {e}") + self.logger.error(f"Command output: {e.stdout}") + self.logger.error(f"Command error: {e.stderr}") + raise RuntimeError(f"Failed to create volume: {e}") + + def _delete_volume(self, path): + """ + Delete a storage volume. + + Args: + path (str): Path to the volume to delete. + + Returns: + bool: True if deletion successful or file doesn't exist, False otherwise. + """ + if not path: + return False + + self.logger.info(f"Deleting volume at path: {path}") + + if not os.path.exists(path): + self.logger.info(f"Volume doesn't exist, nothing to delete: {path}") + return True + + try: + os.remove(path) + self.logger.info(f"Volume deleted successfully: {path}") + return True + except Exception as e: + self.logger.error(f"Failed to delete volume {path}: {e}") + return False + + def _generate_xml_from_config(self): + """ + Generate libvirt XML configuration from virtual_machine_config dictionary. + + Args: + virtual_machine_id (str): A unique identifier for the VirtualMachine to be embedded in the XML. + + Returns: + str: XML configuration for the VirtualMachine. + """ + self.logger.debug(f"Inside _generate_xml_from_config for {self.virtual_machine_id}") + try: + # Validate virtual_machine_config structure + required_config = ["memory", "vcpu"] + for key in required_config: + if key not in self.virtual_machine_config: + raise ValueError(f"Missing required VirtualMachine configuration parameter: {key}") + + # Create root domain element + domain = ET.Element("domain") + domain.set("type", "kvm") # Default to KVirtualMachine, could be made configurable + domain.set("xmlns:custom", "http://example.com/xmlns/libvirt/custom") + + # Basic VirtualMachine information + ET.SubElement(domain, "name").text = self.virtual_machine_id + self.logger.debug(f"Set name") + # Memory configuration (in KiB) + memory = int(self.virtual_machine_config["memory"]) * 1024 # Convert MB to KiB + ET.SubElement(domain, "memory", unit="KiB").text = str(memory) + ET.SubElement(domain, "currentMemory", unit="KiB").text = str(memory) + + # CPU configuration + ET.SubElement(domain, "vcpu", placement="static").text = str(self.virtual_machine_config["vcpu"]) + + # OS configuration + os_elem = ET.SubElement(domain, "os") + type_elem = ET.SubElement(os_elem, "type", arch="x86_64") + type_elem.text = "hvm" + + # Dynamically set machine type + best_machine = self._get_best_machine_type() + type_elem.set("machine", best_machine) + + + # Boot options if specified + if "boot_devices" in self.virtual_machine_config: + for device in self.virtual_machine_config["boot_devices"]: + ET.SubElement(os, "boot", dev=device) + + # Features + features = ET.SubElement(domain, "features") + ET.SubElement(features, "acpi") + ET.SubElement(features, "apic") + ET.SubElement(features, "vmport", state="off") + + # Devices section + devices = ET.SubElement(domain, "devices") + + # Emulator + ET.SubElement(devices, "emulator").text = "/usr/bin/qemu-system-x86_64" + + # Add disks/volumes + if "volumes" in self.virtual_machine_config: + for idx, volume in enumerate(self.virtual_machine_config["volumes"]): + disk = ET.SubElement(devices, "disk", type="file", device="disk") + ET.SubElement(disk, "driver", name="qemu", type="qcow2") + + # Construct the volume path dynamically + volume_id = volume["id"] + format_type = volume.get("format", "qcow2") # Default qcow2 + path = self._get_volume_path(volume_id, format_type) + ET.SubElement(disk, "source", file=path) + + # Determine device name based on index + device_name = f"vd{chr(97 + idx)}" # vda, vdb, etc. + ET.SubElement(disk, "target", dev=device_name, bus="virtio") + + # Add network interfaces + if "networks" in self.virtual_machine_config: + for network in self.virtual_machine_config["networks"]: + interface = ET.SubElement(devices, "interface", type="network") + ET.SubElement(interface, "source", network=network["id"]) + + if "mac_address" in network: + ET.SubElement(interface, "mac", address=network["mac_address"]) + + ET.SubElement(interface, "model", type="virtio") + + # Add graphics if specified + if "graphics" in self.virtual_machine_config: + graphics_config = self.virtual_machine_config["graphics"] + graphics = ET.SubElement(devices, "graphics", type=graphics_config.get("type", "vnc")) + + for attr, value in graphics_config.items(): + if attr != "type": + graphics.set(attr, str(value)) + + # Add metadata section to store VirtualMachine ID and VirtualMachine name + metadata = ET.SubElement(domain, "metadata") + custom_metadata = ET.SubElement(metadata, "custom:metadata", xmlns_custom="http://example.com/xmlns/libvirt/custom") + ET.SubElement(custom_metadata, "custom:virtual_machine_id").text = self.virtual_machine_id + ET.SubElement(custom_metadata, "custom:virtual_machine_name").text = self.virtual_machine_id + + # Convert to pretty XML string + rough_string = ET.tostring(domain, 'utf-8') + reparsed = minidom.parseString(rough_string) + xml_config = reparsed.toprettyxml(indent=" ") + self.logger.info(f"Generated XML configuration for VirtualMachine '{self.virtual_machine_id}' with ID '{self.virtual_machine_id}'.") + return xml_config + + except Exception as e: + self.logger.error(f"Error generating XML configuration: {e}") + raise ValueError(f"Failed to generate XML configuration: {e}") + def execute(self): """ - Execute the task: Manage VM state and optionally perform actions. + Execute the task: Manage VirtualMachine state and optionally perform actions. Returns: dict: Response payload indicating success or failure. @@ -64,87 +414,297 @@ class LibvirtVMTask: if conn is None: raise RuntimeError("Failed to open connection to libvirt.") - # Lookup VM by name + # Extract volume paths from VirtualMachine if it already exists + # This is needed for deletion when the task is called without virtual_machine_config + if not self.volume_paths: + try: + domain = conn.lookupByName(self.virtual_machine_id) + self._extract_volume_paths_from_domain(domain) + except libvirt.libvirtError: + # VirtualMachine doesn't exist, so no volumes to extract + pass + + # Lookup VirtualMachine by name + self.logger.info("Looking up VirtualMachine Name") try: - domain = conn.lookupByName(self.vm_name) - self.logger.info(f"VM '{self.vm_name}' exists. Managing state.") - except libvirt.libvirtError: + # Get all domains (VirtualMachines) managed by the libvirt connection + all_domains = conn.listAllDomains(0) # 0 means no flags, return all domains + self.logger.info(f"Retrieved {len(all_domains)} VirtualMachines from libvirt.") + + # Search for the VirtualMachine with the given name domain = None - self.logger.info(f"VM '{self.vm_name}' does not exist.") + for dom in all_domains: + if dom.name() == self.virtual_machine_id: + domain = dom + self.logger.info(f"VirtualMachine '{self.virtual_machine_id}' exists. Managing state.") + break + + if domain is None: + self.logger.info(f"VirtualMachine '{self.virtual_machine_id}' does not exist.") + except libvirt.libvirtError as e: + self.logger.error(f"Error retrieving domains from libvirt: {e}") + raise e # Handle desired state if self.desired_state == "deleted": if domain: - self.logger.info(f"Deleting VM '{self.vm_name}'.") + self.logger.info(f"Deleting VirtualMachine '{self.virtual_machine_id}'.") if domain.isActive(): - domain.destroy() # Stop the VM if running + domain.destroy() # Stop the VirtualMachine if running domain.undefine() # Remove its definition - response_payload["response"]["status"] = "VM deleted" + + # Delete associated storage volumes + self._delete_storage_volumes() + + response_payload["response"]["status"] = "VirtualMachine deleted" else: - self.logger.info(f"VM '{self.vm_name}' is already deleted.") - response_payload["response"]["status"] = "VM already deleted" + self.logger.info(f"VirtualMachine '{self.virtual_machine_id}' is already deleted.") + response_payload["response"]["status"] = "VirtualMachine already deleted" response_payload["success"] = True elif self.desired_state in ["running", "stopped"]: if domain: - self.logger.info(f"VM '{self.vm_name}' exists. Checking state.") + # Check if VirtualMachine config has changed + if self.xml_config: + current_xml = domain.XMLDesc() + if self._is_xml_config_different(current_xml, self.xml_config): + self.logger.info(f"VirtualMachine configuration has changed. Updating VirtualMachine '{self.virtual_machine_id}'.") + # Need to recreate the VirtualMachine with new configuration + if domain.isActive(): + domain.destroy() # Stop the VirtualMachine if running + domain.undefine() # Remove its definition + domain = conn.defineXML(self.xml_config) # Create with new config + + self.logger.info(f"VirtualMachine '{self.virtual_machine_id}' exists. Checking state.") if self.desired_state == "running": - self.logger.info(f"VM desired state is running, current state is {domain.isActive()}") + self.logger.info(f"VirtualMachine desired state is running, current state is {domain.isActive()}") if not domain.isActive(): - self.logger.info(f"Starting VM '{self.vm_name}'.") - domain.create() # Start the VM - response_payload["response"]["status"] = "VM running" + self.logger.info(f"Starting VirtualMachine '{self.virtual_machine_id}'.") + domain.create() # Start the VirtualMachine + response_payload["response"]["status"] = "VirtualMachine running" else: # desired_state == "stopped" - self.logger.info(f"VM desired state is stopped, current state is {domain.isActive()}") + self.logger.info(f"VirtualMachine desired state is stopped, current state is {domain.isActive()}") if domain.isActive(): - self.logger.info(f"Stopping VM '{self.vm_name}'.") - domain.destroy() # Stop the VM - response_payload["response"]["status"] = "VM stopped" + self.logger.info(f"Stopping VirtualMachine '{self.virtual_machine_id}'.") + domain.destroy() # Stop the VirtualMachine + response_payload["response"]["status"] = "VirtualMachine stopped" else: if not self.xml_config: - raise ValueError(f"XML configuration is required to create VM '{self.vm_name}'.") - self.logger.info(f"Creating VM '{self.vm_name}' with the provided configuration.") + raise ValueError(f"XML configuration is required to create VirtualMachine '{self.virtual_machine_id}'.") + self.logger.info(f"Creating VirtualMachine '{self.virtual_machine_id}' with the provided configuration.") domain = conn.defineXML(self.xml_config) if self.desired_state == "running": - domain.create() # Start the VM - response_payload["response"]["status"] = "VM created and running" + domain.create() # Start the VirtualMachine + response_payload["response"]["status"] = "VirtualMachine created and running" else: - response_payload["response"]["status"] = "VM created and stopped" + response_payload["response"]["status"] = "VirtualMachine created and stopped" response_payload["success"] = True # Handle optional actions if self.action: if not domain: - raise RuntimeError(f"Cannot perform action '{self.action}' on a non-existent VM.") + raise RuntimeError(f"Cannot perform action '{self.action}' on a non-existent VirtualMachine.") if self.action == "start": if not domain.isActive(): - self.logger.info(f"Starting VM '{self.vm_name}'.") + self.logger.info(f"Starting VirtualMachine '{self.virtual_machine_id}'.") domain.create() - response_payload["response"]["action_status"] = "VM started" + response_payload["response"]["action_status"] = "VirtualMachine started" elif self.action == "stop": if domain.isActive(): - self.logger.info(f"Stopping VM '{self.vm_name}'.") + self.logger.info(f"Stopping VirtualMachine '{self.virtual_machine_id}'.") domain.destroy() - response_payload["response"]["action_status"] = "VM stopped" + response_payload["response"]["action_status"] = "VirtualMachine stopped" elif self.action == "reboot": if domain.isActive(): - self.logger.info(f"Rebooting VM '{self.vm_name}'.") + self.logger.info(f"Rebooting VirtualMachine '{self.virtual_machine_id}'.") domain.reboot() - response_payload["response"]["action_status"] = "VM rebooted" + response_payload["response"]["action_status"] = "VirtualMachine rebooted" else: - raise RuntimeError("Cannot reboot a stopped VM.") + raise RuntimeError("Cannot reboot a stopped VirtualMachine.") else: raise ValueError(f"Unsupported action: {self.action}") - self.logger.info(f"Action '{self.action}' completed successfully for VM '{self.vm_name}'.") + self.logger.info(f"Action '{self.action}' completed successfully for VirtualMachine '{self.virtual_machine_id}'.") except Exception as e: - self.logger.error(f"Error executing LibvirtVMTask: {e}") + self.logger.error(f"Error executing LibvirtVirtualMachineTask: {e}") response_payload["response"]["error"] = str(e) finally: if 'conn' in locals() and conn: conn.close() - self.logger.info(f"LibvirtVMTask response: {response_payload}") + self.logger.info(f"LibvirtVirtualMachineTask response: {response_payload}") return response_payload + + def _extract_volume_paths_from_domain(self, domain): + """ + Extract volume paths from an existing domain. + This is useful when deleting a VirtualMachine without having its original configuration. + + Args: + domain: libvirt domain object + """ + self.logger.info(f"Extracting volume paths from existing domain '{self.virtual_machine_id}'") + try: + xml_desc = domain.XMLDesc() + root = ET.fromstring(xml_desc) + + # Find all disk elements that are actually files + for disk in root.findall(".//devices/disk[@type='file']"): + source = disk.find("source") + if source is not None and "file" in source.attrib: + file_path = source.attrib["file"] + self.logger.info(f"Found volume path: {file_path}") + if file_path not in self.volume_paths: + self.volume_paths.append(file_path) + except Exception as e: + self.logger.error(f"Error extracting volume paths: {e}") + + def _delete_storage_volumes(self): + """ + Delete all associated storage volumes. + """ + self.logger.info(f"Deleting storage volumes for VirtualMachine '{self.virtual_machine_id}'") + for path in self.volume_paths: + self._delete_volume(path) + + def _is_xml_config_different(self, current_xml, new_xml): + """ + Compare current VirtualMachine XML with the new XML configuration to determine if update is needed. + + Args: + current_xml (str): Current XML configuration of the VirtualMachine. + new_xml (str): New XML configuration to be applied. + + Returns: + bool: True if configurations are different, False otherwise. + """ + # Use difflib to determine differences between XML configurations + # Note: This is a simple comparison and might not catch all semantic differences + diff = list(difflib.unified_diff( + current_xml.splitlines(), + new_xml.splitlines(), + n=0 + )) + + # Return True if there are differences + return len(diff) > 0 + + + + + # # Example usage + # json_input = ''' + # { + # "url": "https://example.com/path/to/image.jpg.gz", + # "md5": "d41d8cd98f00b204e9800998ecf8427e", + # "compressed": true + # } + # ''' + # logger.info("Starting process_image function...") + # result = process_image(json_input) + # logger.info(f"Result: {result}") + def _process_image(self, json_string): + # Parse the JSON input + self.logger.info("Processing image - Parsing JSON input...") + try: + data = json.loads(json_string) + url = data['url'] + expected_md5 = data['md5'] + is_compressed = data.get('compressed', False) # Default to False if not provided + self.logger.debug(f"URL: {url}, Expected MD5: {expected_md5}, Compressed: {is_compressed}") + except json.JSONDecodeError as e: + self.logger.error(f"Invalid JSON input: {e}") + return f"Error: Invalid JSON input. {e}" + except KeyError as e: + self.logger.error(f"Missing required key in JSON: {e}") + return f"Error: Missing required key in JSON. {e}" + + # Extract the final file name from the URL + file_name = os.path.basename(url) + if is_compressed: + file_name = file_name.rstrip('.gz') # Remove .gz extension if compressed + md5_file_name = f"{file_name}.md5" + self.logger.debug(f"File name: {file_name}, MD5 file name: {md5_file_name}") + + # Check if the image file exists locally + if os.path.exists(file_name): + self.logger.info(f"Image file '{file_name}' exists locally.") + # Check if the MD5 file exists + if os.path.exists(md5_file_name): + self.logger.info(f"MD5 file '{md5_file_name}' exists.") + with open(md5_file_name, 'r') as md5_file: + local_md5 = md5_file.read().strip() + self.logger.debug(f"Local MD5: {local_md5}, Expected MD5: {expected_md5}") + + # Compare the local MD5 with the expected MD5 + if local_md5 == expected_md5: + self.logger.info("MD5 matches. No further action needed.") + return True + else: + self.logger.warning("MD5 mismatch. Deleting local files and downloading again.") + os.remove(file_name) + os.remove(md5_file_name) + else: + self.logger.warning(f"MD5 file '{md5_file_name}' missing. Deleting local image and downloading again.") + os.remove(file_name) + else: + self.logger.info(f"Image file '{file_name}' does not exist locally. Downloading...") + + # Download the image from the URL using streaming + temp_file_name = f"{file_name}.tmp" + try: + self.logger.info(f"Downloading image from URL: {url}") + with requests.get(url, stream=True) as response: + response.raise_for_status() # Raise an error for bad status codes + with open(temp_file_name, 'wb') as temp_file: + for chunk in response.iter_content(chunk_size=8192): + temp_file.write(chunk) + self.logger.info(f"Download complete. Temporary file saved as '{temp_file_name}'.") + + # If the image is compressed, uncompress it + if is_compressed: + self.logger.info(f"Uncompressing the image '{temp_file_name}'...") + with gzip.open(temp_file_name, 'rb') as f_in: + with open(file_name, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + self.logger.info(f"Uncompressed image saved as '{file_name}'.") + os.remove(temp_file_name) # Delete the temporary compressed file + else: + os.rename(temp_file_name, file_name) # Rename the temporary file to the final name + self.logger.info(f"Renamed temporary file '{temp_file_name}' to '{file_name}'.") + + except requests.exceptions.RequestException as e: + self.logger.error(f"Failed to download the image. Error: {e}") + if os.path.exists(temp_file_name): + os.remove(temp_file_name) + return f"Error: Failed to download the image. {e}" + + # Calculate the MD5 hash of the downloaded image + self.logger.info(f"Calculating MD5 hash for '{file_name}'...") + md5_hash = hashlib.md5() + with open(file_name, 'rb') as image_file: + for chunk in iter(lambda: image_file.read(4096), b""): + md5_hash.update(chunk) + actual_md5 = md5_hash.hexdigest() + self.logger.debug(f"Calculated MD5: {actual_md5}, Expected MD5: {expected_md5}") + + # Compare the calculated MD5 with the expected MD5 + if actual_md5 == expected_md5: + self.logger.info("MD5 matches. Saving MD5 hash to file...") + with open(md5_file_name, 'w') as md5_file: + md5_file.write(actual_md5) + self.logger.info(f"MD5 hash saved to '{md5_file_name}'.") + return True + else: + self.logger.error(f"MD5 mismatch. Expected {expected_md5}, got {actual_md5}.") + # Delete the downloaded image and MD5 file if they exist + if os.path.exists(file_name): + os.remove(file_name) + self.logger.info(f"Deleted file '{file_name}'.") + if os.path.exists(md5_file_name): + os.remove(md5_file_name) + self.logger.info(f"Deleted file '{md5_file_name}'.") + return f"Error: MD5 mismatch. Expected {expected_md5}, got {actual_md5}." + \ No newline at end of file diff --git a/workload_states.md b/workload_states.md new file mode 100644 index 0000000..c2da8ca --- /dev/null +++ b/workload_states.md @@ -0,0 +1,8 @@ +States + + +pending-allocation - Workload creeated in DB, has not been dispatched to the worker yet +pending-allocated - Task assigned to a worker and dispatched to the websocket server + +running - Worker has called back the workload is running +deleted - deleted