JamesBhattarai/3cloud-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dns): implement centralized NSController configuration and fix double period bug

Browse Source 
Refactor NSController configuration into a single source of truth module, ensuring consistent DNS volume mounts and resource limits. Fixes bug where trailing periods in universe DNS names caused double periods in VDC zone names. Removes legacy DNS update modes and standardizes on shared volume approach. Adds comprehensive configuration constants and improves worker-side DNS handling.
This commit is contained in:
nexgen_mirrors
2026-01-02 12:58:03 +10:30
parent e5e5f8f58f
commit 1a8b3bbaca
10 changed files with 516 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/controller/api/workload_host_routes.py
+1 -1
View File
@@ -613,7 +613,7 @@ def get_container_workloads_for_host(workload_host_id):
# Get all container workloads for this host
# We need to get pods assigned to this host and then get all containers in those pods
from app.models.models import ContainerPod, Workload
from app.models.models import ContainerPod
from app.utils.create_workload_container import build_pod_payload
# Check for include_deleted query parameter
app/utils/build_pod_payload.py
+27 -23
View File
@@ -92,6 +92,14 @@ def build_pod_payload(pod: ContainerPod, deleted_container_id: Optional[str] = N
if _container.workload_type == "NSController" and "ports" in launch_params:
cont["ports"] = launch_params["ports"]
# Include DNS configuration fields for NSController
if "use_dns" in launch_params:
cont["use_dns"] = launch_params["use_dns"]
if "vdc_id" in launch_params:
cont["vdc_id"] = launch_params["vdc_id"]
if "dns_config" in launch_params:
cont["dns_config"] = launch_params["dns_config"]
# Check if this is a special lifecycle operation (restart)
if _container.status == "pending-restart":
cont["restart"] = True
@@ -148,28 +156,24 @@ def build_pod_payload(pod: ContainerPod, deleted_container_id: Optional[str] = N
def build_nscontroller_launch_params_from_db(nscontroller: Workload) -> Dict:
"""
Build NSController launch params from current database state of port forwardings,
not from stored launch params.
Build NSController launch params from current database state.
This function now delegates to build_nscontroller_config() which is the
single source of truth for NSController configuration. This ensures
NSController containers always have complete configuration including
DNS volumes and resource limits.
Args:
nscontroller: NSController workload instance
Returns:
Complete launch_params dict with all static config and dynamic ports
"""
# Get all active port forwardings for this pod
active_port_forwardings = PortForwarding.query.filter_by(
from app.utils.nscontroller_config import build_nscontroller_config
return build_nscontroller_config(
vdc_id=nscontroller.pod.vdc_id,
container_name=nscontroller.name,
pod_id=nscontroller.pod_id,
deleted=False
).all()
ports = []
for pf in active_port_forwardings:
ports.append({
"internal": pf.internal_port,
"external": pf.external_port,
"internal_ip_address": pf.internal_ip_address,
"external_ip_address": pf.external_ip_address
})
return {
"docker_image": "busybox:latest",
"container_name": nscontroller.name,
"command": "sleep infinity",
"networks": [],
"ports": ports,
}
include_ports=True
)
app/utils/constants.py
+21
View File
@@ -0,0 +1,21 @@
"""
Global configuration constants for xCloudify infrastructure components.
This module centralizes configuration constants to prevent duplication and
make configuration changes easier to manage and audit.
"""
# NSController resource allocation
NSCONTROLLER_CPU_MCPU = 5
NSCONTROLLER_MEMORY_MIB = 199
NSCONTROLLER_IMAGE = "busybox:latest"
NSCONTROLLER_COMMAND = "sleep infinity"
# DNS configuration paths
GLOBAL_DNS_CONFIG_BASE_PATH = "/var/lib/xcloudify/dns-configs"
DNS_CONTAINER_MOUNT_PATH = "/etc/dnsmasq.d"
# Cloudflared sidecar resource allocation (for future use)
CLOUDFLARED_CPU_MCPU = 1
CLOUDFLARED_MEMORY_MIB = 64
CLOUDFLARED_IMAGE = "cloudflare/cloudflared:latest"
app/utils/create_workload_container.py
+15 -26
View File
@@ -38,9 +38,6 @@ from app.services.cloudflare import CloudflareTunnelManager
from app.utils.standard_responses import api_response
from .exceptions import NoFreePortsError, CloudFlareException
# DNS configuration base path for volume mounts
GLOBAL_DNS_CONFIG_BASE_PATH = "/var/lib/xcloudify/dns-configs"
# ─────────────────────────────────────────────────────────────────────────────
# Helper ▸ Validate a single host through the normal filter chain
# ─────────────────────────────────────────────────────────────────────────────
@@ -189,28 +186,16 @@ def persist_pod_and_containers(payload: Dict) -> Dict:
db.session.flush()
created_pod = True
GLOBAL_NSCONTROLLER_CPU_COUNT=5
GLOBAL_NSCONTROLLER_MEM_LIMIT=199
# Create NSController with minimal params and no host
from app.utils.nscontroller_config import build_nscontroller_config
ns_name = f"NSCONTROLLER_{uuid.uuid4()}"
ns_lp = {
"docker_image": "busybox",
"container_name": ns_name,
"command": "sleep infinity",
"networks": [],
"ports": [],
"cpu": GLOBAL_NSCONTROLLER_CPU_COUNT,
"mem_limit": GLOBAL_NSCONTROLLER_MEM_LIMIT,
"volumes": [
{
"type": "bind",
"source": f"{GLOBAL_DNS_CONFIG_BASE_PATH}/vdc-{vdc.id}",
"target": "/etc/dnsmasq.d",
"read_only": True
}
]
}
ns_lp = build_nscontroller_config(
vdc_id=vdc.id,
container_name=ns_name,
pod_id=None, # No pod_id yet at creation time
include_ports=False # Ports added later during allocation
)
nscontroller = Workload(
name=ns_name,
workload_type="NSController",
@@ -224,16 +209,20 @@ def persist_pod_and_containers(payload: Dict) -> Dict:
db.session.flush()
#Create the WorkloadHostPooledResource entry for the NSController
from app.utils.nscontroller_config import get_nscontroller_resource_requirements
resource_requirements = get_nscontroller_resource_requirements()
db.session.add(WorkloadResourceUsage(
resource_type="cpu",
quantity=GLOBAL_NSCONTROLLER_CPU_COUNT,
quantity=resource_requirements["cpu"],
workload_id=nscontroller.id
))
db.session.add(WorkloadResourceUsage(
resource_type="ram",
quantity=GLOBAL_NSCONTROLLER_MEM_LIMIT,
quantity=resource_requirements["ram"],
workload_id=nscontroller.id
))
))
# Audit
AuditEntry.log_event(
app/utils/dns_helpers.py
+20 -15
View File
@@ -18,7 +18,7 @@ from app.models.models import (
PortForwarding, WorkloadHost
)
from app.models.network import NetworkPort, Network, PodDnsRecord
from app.utils.create_workload_container import GLOBAL_DNS_CONFIG_BASE_PATH
from app.utils.constants import GLOBAL_DNS_CONFIG_BASE_PATH
def slugify_name(name: str) -> str:
@@ -57,29 +57,32 @@ def slugify_name(name: str) -> str:
def get_vdc_dns_zone_name(vdc_id: str) -> str:
"""
Get the DNS zone name for a VDC.
Args:
vdc_id: Virtual Data Center ID
Returns:
DNS zone name in format: <vdc-name>.<universe-dns-name>.local
Raises:
ValueError: If VDC not found
"""
vdc = VirtualDataCenter.query.get(vdc_id)
if not vdc:
raise ValueError(f"VDC {vdc_id} not found")
# Get universe for DNS namespace
universe = vdc.project.universe
# Slugify VDC name for DNS compatibility
vdc_slug = slugify_name(vdc.name)
# Strip trailing periods from universe DNS name to avoid double periods
universe_dns_name = universe.universe_dns_name.rstrip('.')
# Build zone name: <vdc-name>.<universe-dns-name>.local
zone_name = f"{vdc_slug}.{universe.universe_dns_name}.local"
zone_name = f"{vdc_slug}.{universe_dns_name}.local"
return zone_name
@@ -148,10 +151,12 @@ def generate_dns_zone_for_vdc(vdc_id: str, logger_instance=None) -> str:
# Get universe for DNS namespace
universe = vdc.project.universe
# Build zone name
vdc_slug = slugify_name(vdc.name)
zone_name = f"{vdc_slug}.{universe.universe_dns_name}.local"
# Strip trailing periods from universe DNS name to avoid double periods
universe_dns_name = universe.universe_dns_name.rstrip('.')
zone_name = f"{vdc_slug}.{universe_dns_name}.local"
records = []
@@ -430,6 +435,7 @@ def send_dns_updates_for_vdc(vdc_id: str, logger_instance=None) -> bool:
).filter(
Workload.workload_host_id.isnot(None) # Only scheduled NSControllers
).all()
logger.debug(f"Found nsconteollers: {nscontrollers}")
# Group NSControllers by host and collect container IDs
# WHY: Workers need container IDs to efficiently locate NSController containers
@@ -437,8 +443,8 @@ def send_dns_updates_for_vdc(vdc_id: str, logger_instance=None) -> bool:
nscontroller_map = {}
for ns in nscontrollers:
host_id = ns.workload_host_id
container_id = ns.container_id
container_id = ns.id
logger.debug (f"Processing NSController {ns.id} on host {host_id} with container ID {container_id}")
if host_id not in nscontroller_map:
nscontroller_map[host_id] = []
@@ -472,8 +478,7 @@ def send_dns_updates_for_vdc(vdc_id: str, logger_instance=None) -> bool:
"vdc_id": vdc_id,
"zone_file": zone_file_content,
"zone_name": zone_name,
"host_directory_path": f"{GLOBAL_DNS_CONFIG_BASE_PATH}/vdc-{vdc_id}",
"nscontroller_container_ids": container_ids # NEW: Container IDs for efficiency
"nscontroller_container_ids": container_ids
}
}
app/utils/nscontroller_config.py
+174
View File
@@ -0,0 +1,174 @@
"""
NSController configuration builder - single source of truth.
This module provides the authoritative NSController configuration,
combining static elements (resources, volumes) with dynamic elements
(ports from database) for both creation and runtime scenarios.
This consolidation fixes critical bugs where NSController loses DNS volume
mounts and resource limits during runtime operations (restart, migration, etc.).
"""
from typing import Dict, List, Optional
from app.models.models import PortForwarding
from app.utils.constants import (
NSCONTROLLER_CPU_MCPU,
NSCONTROLLER_MEMORY_MIB,
NSCONTROLLER_IMAGE,
NSCONTROLLER_COMMAND
)
from app.utils.dns_helpers import generate_dns_zone_for_vdc, get_vdc_dns_zone_name
from app import logger
def build_nscontroller_config(
*,
vdc_id: str,
container_name: str,
pod_id: Optional[str] = None,
include_ports: bool = True
) -> Dict:
"""
Build complete NSController launch parameters.
This is the ONLY function that should create NSController configuration.
It combines static infrastructure elements with dynamic port mappings and DNS configuration.
ARCHITECTURAL CHANGE: DNS volume mounting is now worker-side responsibility.
The server only indicates DNS is needed via use_dns flag and provides the initial
DNS configuration. The worker determines the actual bind mount path based on its
local file system.
Args:
vdc_id: Virtual Data Center ID (required for DNS configuration)
container_name: NSController container name
pod_id: Pod ID (optional, required if include_ports=True)
include_ports: Whether to include port mappings from database
Returns:
Complete launch_params dict ready for JSON serialization with:
- use_dns: Flag indicating DNS should be mounted (worker-side)
- vdc_id: VDC identifier for worker to determine DNS config path
- dns_config: Initial DNS configuration payload (zone_file, zone_name)
- NO volumes array (worker determines bind mount path)
Usage:
# At pod creation (no ports yet):
config = build_nscontroller_config(
vdc_id=vdc.id,
container_name=ns_name,
pod_id=None,
include_ports=False
)
# During runtime operations (with ports):
config = build_nscontroller_config(
vdc_id=vdc.id,
container_name=nscontroller.name,
pod_id=pod.id,
include_ports=True
)
Note:
Single function ensures all NSController instances have identical
static configuration (resources, DNS setup) while allowing dynamic port
injection from database state.
"""
# Generate initial DNS configuration for this VDC
# This provides the NSController with its starting DNS state
dns_config = _generate_dns_config(vdc_id)
# Static configuration - ALWAYS included
# NOTE: No volumes array - worker determines DNS bind mount path
config = {
"docker_image": NSCONTROLLER_IMAGE,
"container_name": container_name,
"command": NSCONTROLLER_COMMAND,
"networks": [],
"cpu": NSCONTROLLER_CPU_MCPU,
"mem_limit": NSCONTROLLER_MEMORY_MIB,
"use_dns": True, # Flag for worker to create DNS bind mount
"vdc_id": vdc_id, # Worker needs this to determine DNS config path
"dns_config": dns_config # Initial DNS configuration payload
}
# Dynamic ports from database (for runtime operations)
if include_ports and pod_id:
config["ports"] = _get_ports_from_db(pod_id)
else:
config["ports"] = []
return config
def _generate_dns_config(vdc_id: str) -> Dict:
"""
Generate initial DNS configuration for NSController.
This creates the DNS payload that NSController needs at startup,
containing the complete DNS zone file and zone name for the VDC.
Args:
vdc_id: Virtual Data Center ID
Returns:
Dict containing:
- zone_file: Complete DNS configuration content
- zone_name: DNS zone name for the VDC
"""
try:
# Generate DNS zone content from current VDC state
zone_file_content = generate_dns_zone_for_vdc(vdc_id, logger)
zone_name = get_vdc_dns_zone_name(vdc_id)
return {
"zone_file": zone_file_content,
"zone_name": zone_name
}
except Exception as e:
logger.error(f"Failed to generate DNS config for VDC {vdc_id}: {e}")
# Return empty DNS config on error - NSController can still start
# but won't have DNS resolution until updated
return {
"zone_file": f"# DNS Zone for VDC {vdc_id}\n# Failed to generate: {str(e)}\n",
"zone_name": f"vdc-{vdc_id}.local"
}
def _get_ports_from_db(pod_id: str) -> List[Dict]:
"""
Fetch current port forwardings from database.
Args:
pod_id: Pod ID to query port forwardings for
Returns:
List of port mapping dicts for NSController configuration
"""
active_port_forwardings = PortForwarding.query.filter_by(
pod_id=pod_id,
deleted=False
).all()
ports = []
for pf in active_port_forwardings:
ports.append({
"internal": pf.internal_port,
"external": pf.external_port,
"internal_ip_address": pf.internal_ip_address,
"external_ip_address": pf.external_ip_address
})
return ports
def get_nscontroller_resource_requirements() -> Dict[str, float]:
"""
Get NSController resource requirements for scheduler.
Returns:
Dict with 'cpu' and 'ram' keys containing resource quantities
"""
return {
"cpu": NSCONTROLLER_CPU_MCPU,
"ram": NSCONTROLLER_MEMORY_MIB
}
test_dns_fix.py
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""
Test script to verify DNS zone name generation fix for double periods.
"""
import sys
import os
# Add the app directory to the path so we can import modules
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'app'))
from app.utils.dns_helpers import get_vdc_dns_zone_name, generate_dns_zone_for_vdc
# Mock the database models for testing
class MockUniverse:
def __init__(self, universe_dns_name):
self.universe_dns_name = universe_dns_name
class MockProject:
def __init__(self, universe):
self.universe = universe
class MockVDC:
def __init__(self, name, project):
self.name = name
self.project = project
def test_get_vdc_dns_zone_name():
"""Test that get_vdc_dns_zone_name strips trailing periods correctly."""
print("Testing get_vdc_dns_zone_name...")
# Test case 1: universe_dns_name with trailing period
universe = MockUniverse("fidget-spinner-research.")
project = MockProject(universe)
vdc = MockVDC("demo-pod", project)
# Mock the query.get method
original_get = None
try:
from app.models.models import VirtualDataCenter
original_get = VirtualDataCenter.query.get
# Mock the get method
VirtualDataCenter.query.get = lambda vdc_id: vdc
zone_name = get_vdc_dns_zone_name("test-vdc-id")
expected = "demo-pod.fidget-spinner-research.local"
if zone_name == expected:
print(f"✓ PASS: {zone_name}")
else:
print(f"✗ FAIL: Expected '{expected}', got '{zone_name}'")
return False
finally:
if original_get:
VirtualDataCenter.query.get = original_get
# Test case 2: universe_dns_name without trailing period
universe2 = MockUniverse("fidget-spinner-research")
project2 = MockProject(universe2)
vdc2 = MockVDC("demo-pod", project2)
try:
VirtualDataCenter.query.get = lambda vdc_id: vdc2
zone_name2 = get_vdc_dns_zone_name("test-vdc-id")
expected2 = "demo-pod.fidget-spinner-research.local"
if zone_name2 == expected2:
print(f"✓ PASS: {zone_name2}")
else:
print(f"✗ FAIL: Expected '{expected2}', got '{zone_name2}'")
return False
finally:
if original_get:
VirtualDataCenter.query.get = original_get
return True
def test_generate_dns_zone_for_vdc():
"""Test that generate_dns_zone_for_vdc strips trailing periods correctly."""
print("\nTesting generate_dns_zone_for_vdc...")
# This is harder to test without a full database setup, so we'll just
# verify the zone name construction part works
universe = MockUniverse("fidget-spinner-research.")
project = MockProject(universe)
vdc = MockVDC("demo-pod", project)
# Test the zone name construction logic from generate_dns_zone_for_vdc
vdc_slug = "demo-pod" # slugify_name would return this
universe_dns_name = universe.universe_dns_name.rstrip('.')
zone_name = f"{vdc_slug}.{universe_dns_name}.local"
expected = "demo-pod.fidget-spinner-research.local"
if zone_name == expected:
print(f"✓ PASS: {zone_name}")
return True
else:
print(f"✗ FAIL: Expected '{expected}', got '{zone_name}'")
return False
if __name__ == "__main__":
print("Testing DNS zone name generation fix...\n")
success1 = test_get_vdc_dns_zone_name()
success2 = test_generate_dns_zone_for_vdc()
if success1 and success2:
print("\n✓ All tests passed! The double period issue should be fixed.")
sys.exit(0)
else:
print("\n✗ Some tests failed!")
sys.exit(1)
worker/settings.py
+1
View File
@@ -33,6 +33,7 @@ class SettingsManager:
"DEFAULT_VOLUME_PATH": "/tmp",
"DEBUG_SOCKETIO": False,
"ENABLE_WEBSOCKET_PING_DEBUG": False,
"GLOBAL_DNS_CONFIG_BASE_PATH": "/var/lib/xcloudify/dns-configs",
"LOG_LEVEL": "INFO",
"NO_DOCKER": False,
"NO_LIBVIRT": False,
worker/worker_tasks/container.py
+94
View File
@@ -12,6 +12,14 @@ from typing import List, Dict, Any, Optional, Callable
from settings import settings
# Import DNS configuration constants
try:
from app.utils.constants import GLOBAL_DNS_CONFIG_BASE_PATH, DNS_CONTAINER_MOUNT_PATH
except ImportError:
# Fallback if app.utils.constants is not available in worker environment
GLOBAL_DNS_CONFIG_BASE_PATH = "/var/lib/xcloudify/dns-configs"
DNS_CONTAINER_MOUNT_PATH = "/etc/dnsmasq.d"
class ContainerTask:
"""
@@ -706,6 +714,92 @@ class ContainerTask:
else:
self.logger.debug(f"No injected_files found in container_spec for {system_container_id}")
# Handle DNS configuration for NSControllers with use_dns flag
use_dns = container_spec.get("use_dns", False)
if use_dns and is_nscontroller:
self.logger.info(f"Handling DNS configuration for NSController {system_container_id}")
# Extract VDC ID from container spec
vdc_id = container_spec.get("vdc_id")
if not vdc_id:
self.logger.error(f"use_dns is true but vdc_id is missing in container spec")
self.launch_failures.append({
"id": system_container_id,
"error": "use_dns requires vdc_id in container specification"
})
return None
# Construct DNS config directory path
dns_config_dir = f"{GLOBAL_DNS_CONFIG_BASE_PATH}"
self.logger.info(f"DNS config directory for VDC {vdc_id}: {dns_config_dir}")
# Create DNS config directory if it doesn't exist
try:
os.makedirs(dns_config_dir, mode=0o755, exist_ok=True)
self.logger.info(f"Ensured DNS config directory exists: {dns_config_dir}")
except Exception as e:
self.logger.error(f"Failed to create DNS config directory {dns_config_dir}: {e}")
self.launch_failures.append({
"id": system_container_id,
"error": f"Failed to create DNS config directory: {str(e)}"
})
return None
# Write initial DNS configuration if provided
dns_config = container_spec.get("dns_config")
if dns_config:
zone_file = dns_config.get("zone_file")
zone_name = dns_config.get("zone_name")
if zone_file and zone_name:
config_filename = f"{vdc_id}.conf"
config_path = os.path.join(dns_config_dir, config_filename)
try:
# Write DNS config atomically using temp file
temp_path = config_path + ".tmp"
with open(temp_path, 'w') as f:
f.write(zone_file)
if not zone_file.endswith('\n'):
f.write('\n')
f.flush()
os.fsync(f.fileno())
# Set proper permissions before rename
os.chmod(temp_path, 0o644)
# Atomic rename
os.rename(temp_path, config_path)
self.logger.info(f"Initial DNS config written to {config_path}")
except Exception as e:
# Clean up temp file if it exists
if os.path.exists(temp_path):
try:
os.unlink(temp_path)
except Exception:
pass
self.logger.error(f"Failed to write initial DNS config: {e}")
# Don't fail container creation, just log the error
# DNS can be updated later via DNS update task
else:
self.logger.debug("dns_config provided but missing zone_file or zone_name")
else:
self.logger.debug("No initial dns_config provided in container spec")
# Add DNS bind mount to container volumes
if "volumes" not in container_kwargs:
container_kwargs["volumes"] = {}
container_kwargs["volumes"][dns_config_dir] = {
"bind": DNS_CONTAINER_MOUNT_PATH,
"mode": "ro" # Read-only mount for security
}
self.logger.info(
f"Added DNS bind mount: {dns_config_dir} -> {DNS_CONTAINER_MOUNT_PATH} (read-only)"
)
try:
# Step 1 – create & start
container = self.docker_client.containers.run(**container_kwargs)
worker/worker_tasks/dns.py
+47 -331
View File
@@ -5,10 +5,12 @@ Handles DNS zone configuration updates by writing dnsmasq configuration
files to NSController containers and reloading the DNS service.
"""
import docker
import tempfile
import os
from typing import Dict, Any, Optional
# Import settings for configuration
from settings import settings
class DnsTask:
"""
@@ -32,60 +34,60 @@ class DnsTask:
def handle_dns_update(self, job_details: Dict[str, Any]) -> Dict[str, Any]:
"""
Process DNS zone update with dual-mode support.
Supports two modes:
1. Shared Volume Mode (PREFERRED): Write once to host directory, reload all containers
2. Legacy Mode (FALLBACK): Write to each container individually via docker exec
Process DNS zone update using shared volume mode only.
Requires VDC ID and NSController container IDs. Derives the DNS config path
from VDC ID using the pattern: {settings.get_value('GLOBAL_DNS_CONFIG_BASE_PATH')}/vdc-{vdc_id}
Args:
job_details: {
"vdc_id": "uuid-of-vdc",
"zone_file": "address=/pod1.production-vdc.mycloud.local/10.0.1.5\n...",
"vdc_id": "uuid-of-vdc" (required),
"zone_file": "address=/pod1.production-vdc.mycloud.local/10.0.1.5\n..." (required),
"zone_name": "production-vdc.mycloud.local" (optional, will parse from zone_file if not provided),
"host_directory_path": "/var/lib/xcloudify/dns-configs/vdc-{vdc_id}/" (optional, enables shared volume mode),
"nscontroller_container_ids": ["container_id_1", "container_id_2"] (optional)
"nscontroller_container_ids": ["container_id_1", "container_id_2"] (required)
}
Returns:
dict: {
"success": bool,
"config_path": str,
"lines_written": int,
"containers_updated": int or "containers_reloaded": int,
"mode": "shared_volume" or "legacy",
"containers_reloaded": int,
"mode": "shared_volume",
"error": str (if failed)
}
Mode Selection:
- Tries shared volume mode if both host_directory_path and nscontroller_container_ids are provided
- Falls back to legacy mode if:
* host_directory_path is missing
* nscontroller_container_ids is missing
* Shared volume mode fails
"""
self.logger.info("Processing DNS update task with dual-mode support")
self.logger.info("Processing DNS update task (shared volume mode only)")
try:
# Validate required fields
vdc_id = job_details.get("vdc_id")
zone_file = job_details.get("zone_file")
host_directory_path = job_details.get("host_directory_path")
nscontroller_container_ids = job_details.get("nscontroller_container_ids")
zone_name = job_details.get("zone_name")
if not vdc_id:
return {
"success": False,
"error": "Missing vdc_id in job_details"
}
if not zone_file:
return {
"success": False,
"error": "Missing zone_file in job_details"
}
if not nscontroller_container_ids:
return {
"success": False,
"error": "Missing nscontroller_container_ids in job_details (required for shared volume mode)"
}
# Derive host directory path from VDC ID
host_directory_path = f"{settings.get_value('GLOBAL_DNS_CONFIG_BASE_PATH')}"
self.logger.info(f"Derived DNS config path from VDC ID: {host_directory_path}")
# Parse zone name from zone file if not provided
if not zone_name:
zone_name = self._parse_zone_name_from_file(zone_file)
@@ -94,50 +96,23 @@ class DnsTask:
"success": False,
"error": "Could not parse zone name from zone_file"
}
self.logger.info(f"Updating DNS for VDC {vdc_id}, zone: {zone_name}")
# Try shared volume mode first if both required fields are provided
if host_directory_path and nscontroller_container_ids:
self.logger.info("Attempting shared volume mode")
result = self._write_to_shared_volume(
host_directory_path=host_directory_path,
zone_name=zone_name,
zone_file=zone_file,
nscontroller_container_ids=nscontroller_container_ids
)
if result["success"]:
self.logger.info("Shared volume mode succeeded")
return result
else:
self.logger.warning(
f"Shared volume mode failed: {result.get('error')}. "
"Falling back to legacy mode"
)
else:
# Log why shared volume mode is not being used
missing_fields = []
if not host_directory_path:
missing_fields.append("host_directory_path")
if not nscontroller_container_ids:
missing_fields.append("nscontroller_container_ids")
self.logger.info(
f"Shared volume mode not available (missing: {', '.join(missing_fields)}). "
"Using legacy mode"
)
# Fall back to legacy mode
self.logger.info("Using legacy mode")
result = self._write_to_containers_legacy(
vdc_id=vdc_id,
zone_name=zone_name,
# Use shared volume mode only
self.logger.info("Using shared volume mode")
result = self._write_to_shared_volume(
host_directory_path=host_directory_path,
zone_id=vdc_id,
zone_file=zone_file,
nscontroller_container_ids=nscontroller_container_ids
)
if result["success"]:
self.logger.info("Shared volume mode succeeded")
else:
self.logger.error(f"Shared volume mode failed: {result.get('error')}")
return result
except Exception as e:
@@ -193,134 +168,6 @@ class DnsTask:
self.logger.error(f"Error parsing zone name from file: {str(e)}")
return None
def _find_nscontroller_for_vdc(self, vdc_id: str) -> Optional[docker.models.containers.Container]:
"""
Find NSController container on this host for the VDC.
Searches Docker containers with label managed_by=<worker_id> and
filters for NSController workload_type containers associated with
the specified VDC.
Args:
vdc_id: Virtual Data Center ID
Returns:
Docker container object or None if not found
"""
try:
# Get all containers managed by this worker
filters = {"label": f"managed_by={self.worker_id}"} if self.worker_id else {}
containers = self.docker_client.containers.list(all=True, filters=filters)
self.logger.debug(f"Searching for NSController among {len(containers)} containers")
# Find NSController for this VDC
for container in containers:
labels = container.labels or {}
# Check if this is an NSController
# NSControllers are identified by having workload_type label
workload_type = labels.get("workload_type")
container_vdc_id = labels.get("vdc_id")
if workload_type == "NSController" and container_vdc_id == vdc_id:
self.logger.info(f"Found NSController: {container.name} for VDC {vdc_id}")
return container
self.logger.warning(f"No NSController found for VDC {vdc_id}")
return None
except Exception as e:
self.logger.error(f"Error finding NSController: {str(e)}")
return None
def _write_dns_config(self, nscontroller_container: docker.models.containers.Container,
zone_name: str, config_content: str) -> bool:
"""
Write DNS configuration to NSController container.
Uses docker exec to write file inside container atomically:
1. Create temp file with content
2. Write to container using docker cp
3. Move to final location inside container
Args:
nscontroller_container: NSController Docker container
zone_name: DNS zone name (used for filename)
config_content: Complete dnsmasq configuration content
Returns:
bool: True if successful, False otherwise
"""
try:
config_filename = f"{zone_name}.conf"
container_config_path = f"{self.config_dir}/{config_filename}"
self.logger.debug(f"Writing DNS config to {container_config_path}")
# Create temporary file with config content
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.conf') as tmp_file:
tmp_file.write(config_content)
tmp_file.write('\n') # Ensure newline at end
tmp_file.flush()
tmp_path = tmp_file.name
try:
# Ensure the dnsmasq.d directory exists in container
exec_result = nscontroller_container.exec_run(
f"mkdir -p {self.config_dir}",
user='root'
)
if exec_result.exit_code != 0:
self.logger.warning(f"mkdir command output: {exec_result.output.decode()}")
# Method 1: Try using docker cp (preferred)
try:
# Read the temp file content
with open(tmp_path, 'r') as f:
content = f.read()
# Write directly using exec_run with shell redirection
# Escape single quotes in content
escaped_content = content.replace("'", "'\\''")
exec_cmd = f"sh -c 'cat > {container_config_path} << \"EOF\"\n{content}\nEOF'"
exec_result = nscontroller_container.exec_run(
exec_cmd,
user='root'
)
if exec_result.exit_code == 0:
self.logger.info(f"DNS config written to {container_config_path}")
# Set proper permissions
nscontroller_container.exec_run(
f"chmod 644 {container_config_path}",
user='root'
)
return True
else:
self.logger.error(
f"Failed to write config via exec: {exec_result.output.decode()}"
)
return False
except Exception as cp_error:
self.logger.error(f"Error writing config: {str(cp_error)}")
return False
finally:
# Clean up temp file
try:
os.unlink(tmp_path)
except Exception:
pass
except Exception as e:
self.logger.error(f"Error writing DNS config: {str(e)}", exc_info=True)
return False
def _reload_dnsmasq(self, nscontroller_container: docker.models.containers.Container) -> bool:
"""
Reload dnsmasq in NSController via SIGHUP.
@@ -394,7 +241,7 @@ class DnsTask:
self.logger.error(f"Error reloading dnsmasq: {str(e)}", exc_info=True)
return False
def _write_to_shared_volume(self, host_directory_path: str, zone_name: str,
def _write_to_shared_volume(self, host_directory_path: str, zone_id: str,
zone_file: str, nscontroller_container_ids: list) -> Dict[str, Any]:
"""
Write DNS zone file to shared host directory with atomic operation.
@@ -403,8 +250,8 @@ class DnsTask:
Writes once to the host filesystem, then triggers reload in all containers.
Args:
host_directory_path: Host directory path (e.g., /var/lib/xcloudify/dns-configs/vdc-{vdc_id}/)
zone_name: DNS zone name (used for filename)
host_directory_path: Host directory path (e.g., /var/lib/xcloudify/dns-configs/)
zone_id: ID of the VDC (used for filename)
zone_file: Complete dnsmasq configuration content
nscontroller_container_ids: List of NSController container IDs to reload
@@ -434,7 +281,7 @@ class DnsTask:
}
# Prepare file paths
config_filename = f"{zone_name}.conf"
config_filename = f"{zone_id}.conf"
final_path = os.path.join(host_directory_path, config_filename)
temp_path = final_path + ".tmp"
@@ -538,135 +385,4 @@ class DnsTask:
except Exception as e:
self.logger.error(f"Error reloading container {container_id}: {str(e)}")
return successful_reloads
def _write_to_containers_legacy(self, vdc_id: str, zone_name: str, zone_file: str,
nscontroller_container_ids: Optional[list] = None) -> Dict[str, Any]:
"""
Write DNS configuration to containers using legacy docker exec method.
This is the fallback method when shared volume mode is not available.
Writes to each container individually via docker exec.
Args:
vdc_id: Virtual Data Center ID
zone_name: DNS zone name
zone_file: Complete dnsmasq configuration content
nscontroller_container_ids: Optional list of container IDs (will discover if not provided)
Returns:
dict: {
"success": bool,
"config_path": str,
"lines_written": int,
"containers_updated": int,
"mode": "legacy",
"error": str (if failed)
}
"""
try:
self.logger.info("Using legacy mode: writing to containers individually")
# Get NSController containers
containers = []
if nscontroller_container_ids:
# Use provided container IDs (optimized path)
self.logger.info(
f"Using provided NSController container IDs: {nscontroller_container_ids}"
)
for container_id in nscontroller_container_ids:
try:
container = self.docker_client.containers.get(container_id)
containers.append(container)
self.logger.debug(f"Retrieved container: {container.name} ({container_id})")
except docker.errors.NotFound:
self.logger.warning(f"Container {container_id} not found, skipping")
except Exception as e:
self.logger.error(f"Error retrieving container {container_id}: {str(e)}")
if not containers:
return {
"success": False,
"error": f"None of the provided container IDs were found on this worker",
"mode": "legacy"
}
else:
# Fallback to discovery method (backwards compatibility)
self.logger.info(
f"No container IDs provided, falling back to discovery for VDC {vdc_id}"
)
nscontroller = self._find_nscontroller_for_vdc(vdc_id)
if not nscontroller:
return {
"success": False,
"error": f"No NSController found for VDC {vdc_id}",
"mode": "legacy"
}
containers.append(nscontroller)
self.logger.info(f"Found NSController container via discovery: {nscontroller.name}")
# Process each container
successful_updates = 0
config_path = f"{self.config_dir}/{zone_name}.conf"
for container in containers:
self.logger.info(f"Updating DNS config in container: {container.name}")
# Write DNS configuration to NSController container
write_success = self._write_dns_config(container, zone_name, zone_file)
if not write_success:
self.logger.error(
f"Failed to write DNS configuration to container {container.name}"
)
continue
# Reload dnsmasq to apply changes
reload_success = self._reload_dnsmasq(container)
if not reload_success:
self.logger.error(
f"Failed to reload dnsmasq in container {container.name}"
)
continue
successful_updates += 1
self.logger.info(f"Successfully updated DNS in container {container.name}")
# Determine overall success
if successful_updates == 0:
return {
"success": False,
"error": "Failed to update DNS in any NSController container",
"config_path": config_path,
"mode": "legacy"
}
# Count lines written
lines_written = zone_file.count('\n') + 1
self.logger.info(
f"Legacy DNS update completed: "
f"{lines_written} lines written to {config_path} in {successful_updates} container(s)"
)
return {
"success": True,
"config_path": config_path,
"lines_written": lines_written,
"containers_updated": successful_updates,
"total_containers": len(containers),
"mode": "legacy"
}
except Exception as e:
self.logger.error(f"Error in legacy write: {str(e)}", exc_info=True)
return {
"success": False,
"error": str(e),
"mode": "legacy"
}
return successful_reloads