From 35f57ddb79ac511a4cca4f8589ff8dc40d2cb8e1 Mon Sep 17 00:00:00 2001 From: Cory Hawkvelt Date: Mon, 7 Jul 2025 15:12:34 +0930 Subject: [PATCH] Added docker-compose capability --- .dockerignore | 49 ++++++ .gitignore | 3 +- Dockerfile | 29 ++++ app/controller/__init__.py | 5 + app/controller/api/api_routes.py | 2 +- .../api/workload_container_routes.py | 2 +- docker-compose.yml | 146 ++++++++++++++++++ requirements.txt | 22 +-- vnc_proxy.py | 64 +++++--- websocket_server/config.py | 2 +- websocket_server/main.py | 3 +- 11 files changed, 287 insertions(+), 40 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7517d1d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,49 @@ +# Python artifacts +__pycache__/ +*.pyc +*.pyo +*.pyd + +# Virtual environments +.venv/ +env/ +venv/ + +# IDE/editor files +.vscode/ +.idea/ + +# Git and build artifacts +.git/ +.gitignore +*.log +*.out +*.bak +*.swp +*.tmp + +# OS junk +.DS_Store +Thumbs.db + +# Notebooks or datasets (optional) +*.ipynb +data/ +notebooks/ + +# Docker & Compose +docker-compose.override.yml +*.tar +*.gz + +__pycache__ +instance/* +logs/* +.venv/ +mariadb_data/ +.env +.aider* +worker_settings.json +worker/logs/ +celerybeat-schedule +logs/ diff --git a/.gitignore b/.gitignore index 21a7fa1..4d5b3de 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ mariadb_data/ .aider* worker_settings.json worker/logs/ -celerybeat-schedule \ No newline at end of file +celerybeat-schedule +logs/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4c4ccfd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.11-slim + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + POETRY_VERSION=1.7.1 + +# Set working directory +WORKDIR /app + +# Install OS dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + libmariadb-dev \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements and install +COPY requirements.txt . + +RUN pip install --upgrade pip \ + && pip install -r requirements.txt + +# Copy project files +COPY . . + +# Default entrypoint (can be overridden in docker-compose) +CMD ["bash"] diff --git a/app/controller/__init__.py b/app/controller/__init__.py index 39a2fd2..c99cea4 100644 --- a/app/controller/__init__.py +++ b/app/controller/__init__.py @@ -27,3 +27,8 @@ from .api.workload_vm_routes import * from .api.virtual_data_center_routes import * from .api.permission_routes import * + + +@api_bp.route('/healthz', methods=['GET']) +def healthz_check(): + return jsonify("ok") \ No newline at end of file diff --git a/app/controller/api/api_routes.py b/app/controller/api/api_routes.py index e4b1097..a1f3330 100644 --- a/app/controller/api/api_routes.py +++ b/app/controller/api/api_routes.py @@ -67,4 +67,4 @@ create_routes(WorkloadHostPooledResource, 'workload_host_pooled_resources') create_routes(Label, 'labels') create_routes(User, 'users') # create_routes(RegionAccess, 'region_access') -# create_routes(Workload, 'workloads') \ No newline at end of file +# create_routes(Workload, 'workloads') diff --git a/app/controller/api/workload_container_routes.py b/app/controller/api/workload_container_routes.py index ff3b45e..4b92075 100644 --- a/app/controller/api/workload_container_routes.py +++ b/app/controller/api/workload_container_routes.py @@ -170,7 +170,7 @@ def request_container_workload(): def update_container_workload(system_container_id): data = request.json logger.debug(data) - clok here - need to add error to auti table if there is one + # TODO - need to add error to autid table if there is one # Validate new_status valid_statuses = ["running", "deleted", "stopped","dead", "launch_failed"] new_status=data.get("new_status") diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c25ea3f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,146 @@ +version: '3.8' + +services: + + mariadb: + image: mariadb:10.11 + container_name: mariadb + environment: + MYSQL_ROOT_PASSWORD: password + MYSQL_DATABASE: theapi + MYSQL_USER: theapi_user + MYSQL_PASSWORD: theapi_password + ports: + - "3306:3306" + volumes: + - mariadb_data:/var/lib/mysql + - ./initdb:/docker-entrypoint-initdb.d + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-ppassword"] + interval: 10s + timeout: 5s + retries: 5 + + + redis-server: + image: redis + container_name: redis-server + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + + novnc: + image: theasp/novnc + container_name: novnc + ports: + - "6081:6080" + - "8080:8080" + + + api-server: + build: + context: . + container_name: api-server + volumes: + - .:/app + working_dir: /app + command: > + sh -c "flask run --host=0.0.0.0 --port=5000 --debug" + ports: + - "5000:5000" + environment: + FLASK_APP: api_server.py + FLASK_ENV: development + depends_on: + - mariadb + - redis-server + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5000/api/healthz"] + interval: 10s + timeout: 5s + retries: 5 + + + websocket-server: + build: + context: . + container_name: websocket-server + command: python3 -m websocket_server.main + ports: + - "6001:6001" + volumes: + - .:/app + working_dir: /app + depends_on: + - mariadb + - redis-server + + + vnc-proxy: + build: + context: . + container_name: vnc-proxy + command: python3 vnc_proxy.py + ports: + - "6002:6002" + volumes: + - .:/app + working_dir: /app + depends_on: + - websocket-server + environment: + WSS_SERVER_URL: http://172.17.0.1:6001 + CHECK_PERMISSION_URL: http://172.17.0.1:5000/api/check_permission + VNC_PROXY_HTTP_PORT: 6002 + ENGINEIO_LOGGER_LEVEL: debug + SIO_LOGGER_LEVEL: debug + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:6002/healthz"] + interval: 10s + timeout: 5s + retries: 5 + + + + streamlit-server: + build: + context: . + container_name: streamlit-server + command: streamlit run streamlit_server.py --server.port=8501 --server.address=0.0.0.0 + ports: + - "8501:8501" + volumes: + - .:/app + working_dir: /app + + celery-worker: + build: + context: . + container_name: celery-worker + command: celery -A app.celery_app.celery worker --loglevel=info + volumes: + - .:/app + working_dir: /app + depends_on: + - redis-server + - mariadb + + celery-beat: + build: + context: . + container_name: celery-beat + command: celery -A app.celery_app.celery beat --loglevel=info + volumes: + - .:/app + working_dir: /app + depends_on: + - redis-server + - mariadb + +volumes: + mariadb_data: diff --git a/requirements.txt b/requirements.txt index b0c6d2d..a4d582c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,27 +1,17 @@ flask -python-socketio colorlog +python-socketio flask_socketio +websocket-client +flask_sqlalchemy +flask_migrate requests sqlalchemy redis pymysql -flask_sqlalchemy -flask_migrate -websocket-client streamlit aiohttp -#sudo apt install python3-rados python3-rbd librados-dev librbd-dev -# ln -s /usr/lib/python3/dist-packages/rados* $(python -c 'import site; print(site.getsitepackages()[0])')/ -# ln -s /usr/lib/python3/dist-packages/rbd* $(python -c 'import site; print(site.getsitepackages()[0])')/ - - -# sudo apt install -y libvirt-dev python3-dev gcc -libvirt-python # I think requires libvirt-dev? -# Alternativly apt-get install libvirt-dev - -netifaces jwt -docker python-dotenv -psutil \ No newline at end of file +psutil +celery \ No newline at end of file diff --git a/vnc_proxy.py b/vnc_proxy.py index 452c6fa..b052ac5 100644 --- a/vnc_proxy.py +++ b/vnc_proxy.py @@ -7,19 +7,27 @@ import uuid from aiohttp import web import requests import socketio +import os # Config -HTTP_PORT = 6002 -VNC_PORT_LOCAL = 5900 -WSS_Server_URL="http://127.0.0.1:6001" +# Config from environment +HTTP_PORT = int(os.getenv("VNC_PROXY_HTTP_PORT", 6002)) +VNC_PORT_LOCAL = int(os.getenv("VNC_PORT_LOCAL", 5900)) +WSS_Server_URL = os.getenv("WSS_SERVER_URL", "http://172.17.0.1:6001") +CHECK_PERMISSION_URL = os.getenv("CHECK_PERMISSION_URL", "http://localhost:5000/api/check_permission") +ENGINEIO_LOGGER_LEVEL = os.getenv("ENGINEIO_LOGGER_LEVEL", "WARNING").upper() +SIO_LOGGER_LEVEL = os.getenv("SIO_LOGGER_LEVEL", "WARNING").upper() + logging.basicConfig(level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("middleware") -# Configure socketio and engineio loggers to use WARNING level to reduce verbosity +# Apply dynamic log levels from environment sio_logger = logging.getLogger("socketio.client") -sio_logger.setLevel(logging.INFO) +sio_logger.setLevel(getattr(logging, SIO_LOGGER_LEVEL, logging.WARNING)) + engineio_logger = logging.getLogger("engineio.client") -engineio_logger.setLevel(logging.INFO) +engineio_logger.setLevel(getattr(logging, ENGINEIO_LOGGER_LEVEL, logging.WARNING)) + # sio = socketio.AsyncServer(async_mode="aiohttp", cors_allowed_origins="*") app = web.Application() @@ -28,7 +36,6 @@ sio = socketio.AsyncClient(logger=sio_logger, engineio_logger=engineio_logger) sessions = {} # vnc_request_id → WebSocket routes = web.RouteTableDef() -CHECK_PERMISSION_URL = "http://localhost:5000/api/check_permission" # Adjust to actual location @sio.event @@ -167,23 +174,42 @@ async def vnc_handler(request): await ws.close() return ws + +@routes.get("/healthz") +async def health_check(request): + """ + Healthcheck endpoint used for container liveness checks. + Returns HTTP 200 with a simple message. + """ + return web.Response(status=200, text="ok") + + app.add_routes(routes) +async def connect_with_retry(max_retries=10): + delay = 1 + for attempt in range(1, max_retries + 1): + try: + logger.info(f"Connecting to WebSocket server (attempt {attempt})...") + await sio.connect(WSS_Server_URL) + logger.info("WebSocket connection established") + return + except Exception as e: + logger.warning(f"WebSocket connection failed: {e}") + if attempt == max_retries: + logger.error("Max retries exceeded. Giving up.") + raise SystemExit(1) + await asyncio.sleep(delay) + delay *= 2 + async def main(): runner = web.AppRunner(app) await runner.setup() - - await sio.connect(WSS_Server_URL) - # site1 = web.TCPSite(runner, host="0.0.0.0", port=SOCKETIO_PORT) - site2 = web.TCPSite(runner, host="0.0.0.0", port=HTTP_PORT) - - # await site1.start() - await site2.start() - - # logger.info(f"Socket.IO running on http://localhost:{SOCKETIO_PORT}") - logger.info(f"NoVNC WebSocket at ws://localhost:{HTTP_PORT}/vnc") - + await connect_with_retry() + site = web.TCPSite(runner, host="0.0.0.0", port=HTTP_PORT) + await site.start() + logger.info(f"NoVNC WebSocket at ws://0.0.0.0:{HTTP_PORT}/vnc") await asyncio.Event().wait() if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) \ No newline at end of file diff --git a/websocket_server/config.py b/websocket_server/config.py index b0382a1..03cca29 100644 --- a/websocket_server/config.py +++ b/websocket_server/config.py @@ -14,7 +14,7 @@ REDIS_HOST = "localhost" REDIS_PORT = 6379 # Database configuration -DATABASE_URL = "mysql://root:password@172.17.0.1:3306/defaultdb" +DATABASE_URL = "mysql://root:password@172.17.0.1:3306/theapi" # Connection pool for Redis redis_connection_pool = redis.ConnectionPool( diff --git a/websocket_server/main.py b/websocket_server/main.py index 72717b7..5807504 100644 --- a/websocket_server/main.py +++ b/websocket_server/main.py @@ -50,4 +50,5 @@ def start_background_threads(): if __name__ == "__main__": start_background_threads() - base.socketio.run(app, host="0.0.0.0", port=6001) + # TODO - Remove allow_unsafe_werkzeug + base.socketio.run(app, host="0.0.0.0", port=6001,allow_unsafe_werkzeug=True )