From 7ec2e4d47b29415afb5c4e7b4396f8f12e46ff4d Mon Sep 17 00:00:00 2001 From: James Bhattarai Date: Sat, 8 Aug 2026 15:29:10 +0200 Subject: [PATCH] Feat: Implement NAT for Private network Added enable_nat bool for networks to allow nats Implemented docker network bridge, to allow nat with tenancy seperated as `enable_icc:false` --- app/controller/api/network_routes.py | 2 + app/controller/api/workload_host_routes.py | 1 + app/models/network.py | 1 + app/utils/build_pod_payload.py | 23 ++++--- app/utils/constants.py | 4 ++ app/utils/nscontroller_config.py | 3 + app/utils/nscontroller_helpers.py | 1 + .../versions/0003_network_enable_host_nat.py | 23 +++++++ sandbox/containers/nscontroller/dnsmasq.conf | 6 ++ sandbox/containers/nscontroller/entrypoint.sh | 17 +++++ worker/worker_tasks/container.py | 65 +++++++++++++++++-- 11 files changed, 131 insertions(+), 15 deletions(-) create mode 100644 migrations/versions/0003_network_enable_host_nat.py diff --git a/app/controller/api/network_routes.py b/app/controller/api/network_routes.py index 8a00485..ffaa8e1 100644 --- a/app/controller/api/network_routes.py +++ b/app/controller/api/network_routes.py @@ -61,6 +61,7 @@ def validate_network_data(data, is_update=False): 'ipv6_cidr': str, 'ipv6_gateway': str, 'ovs_bridge': str, + 'enable_host_nat': bool, 'encapsulation': str, } for field, field_type in optional_fields.items(): @@ -139,6 +140,7 @@ def add_network(): ipv6_cidr=validated_data.get('ipv6_cidr'), ipv6_gateway=validated_data.get('ipv6_gateway'), ovs_bridge=validated_data.get('ovs_bridge'), + enable_host_nat=validated_data.get('enable_host_nat', False), **({'encapsulation': validated_data['encapsulation']} if 'encapsulation' in validated_data else {}), ) db.session.add(instance) diff --git a/app/controller/api/workload_host_routes.py b/app/controller/api/workload_host_routes.py index dcd3d8c..8744477 100644 --- a/app/controller/api/workload_host_routes.py +++ b/app/controller/api/workload_host_routes.py @@ -784,6 +784,7 @@ def get_container_workloads_for_host(workload_host_id): "mem_limit": nsc_config.get("mem_limit", 128), "pod_id": f"standalone-nsc-{nsc.id}", "network_ports": [], # Initialize as empty; will populate if port exists + "enable_host_nat": nsc_config.get("enable_host_nat", False), } if port: diff --git a/app/models/network.py b/app/models/network.py index d32bc3e..191269e 100644 --- a/app/models/network.py +++ b/app/models/network.py @@ -78,6 +78,7 @@ class Network(BaseModel): encapsulation = Column(String(50), default="VXLAN", nullable=False) ovs_bridge = Column(String(255), nullable=True) network_type = Column(String(255), nullable=True) + enable_host_nat = Column(Boolean, default=False, nullable=False, server_default="0") def __init__(self, *args, **kwargs): # Convert UUID strings to UUID objects for any UUID fields diff --git a/app/utils/build_pod_payload.py b/app/utils/build_pod_payload.py index 9c6cda3..df4ecb0 100644 --- a/app/utils/build_pod_payload.py +++ b/app/utils/build_pod_payload.py @@ -91,17 +91,20 @@ def build_pod_payload(pod: ContainerPod, deleted_container_id: Optional[str] = N } # Include ports for NSController - if _container.workload_type == "NSController" and "ports" in launch_params: - cont["ports"] = launch_params["ports"] + if _container.workload_type == "NSController": + cont["enable_host_nat"] = launch_params.get("enable_host_nat", False) + + if "ports" in launch_params: + cont["ports"] = launch_params["ports"] + + # Include DNS configuration fields for NSController + if "use_dns" in launch_params: + cont["use_dns"] = launch_params["use_dns"] + if "vdc_id" in launch_params: + cont["vdc_id"] = launch_params["vdc_id"] + if "dns_config" in launch_params: + cont["dns_config"] = launch_params["dns_config"] - # Include DNS configuration fields for NSController - if "use_dns" in launch_params: - cont["use_dns"] = launch_params["use_dns"] - if "vdc_id" in launch_params: - cont["vdc_id"] = launch_params["vdc_id"] - if "dns_config" in launch_params: - cont["dns_config"] = launch_params["dns_config"] - # Check if this is a special lifecycle operation (restart) if _container.status == "pending-restart": cont["restart"] = True diff --git a/app/utils/constants.py b/app/utils/constants.py index a5e4380..da95df0 100644 --- a/app/utils/constants.py +++ b/app/utils/constants.py @@ -28,3 +28,7 @@ CLOUDFLARED_IMAGE = "cloudflare/cloudflared:latest" # Do not hard-code any fallback here; if not set, payload builders must omit # the ovs_bridge key when a network does not define it. DEFAULT_OVS_BRIDGE = os.getenv('XCF_DEFAULT_OVS_BRIDGE') + +# Docker bridge network NSControllers attach to when a network has +# enable_host_nat set, for internet egress. +NSCONTROLLER_NAT_NETWORK = "xcloudify-wan" diff --git a/app/utils/nscontroller_config.py b/app/utils/nscontroller_config.py index 737ff63..aef30b3 100644 --- a/app/utils/nscontroller_config.py +++ b/app/utils/nscontroller_config.py @@ -111,6 +111,8 @@ def build_nscontroller_config( "network_id": network_id, # Worker/proxy uses this to scope metadata lookups "dns_config": dns_config, # Initial DNS configuration payload "env": env, # Dynamic provider network environment variables + # To allow nat via Docker-bridge uplink + "enable_host_nat": bool(provider_net.enable_host_nat), } # Dynamic ports from database (for runtime operations) @@ -202,6 +204,7 @@ def _build_env(vdc_id: str, network_id: str) -> Dict[str, str]: env["PROVIDER_DHCP_RANGE_START"] = provider_net.dhcp_range_start or "" env["PROVIDER_DHCP_RANGE_END"] = provider_net.dhcp_range_end or "" env["PROVIDER_DOMAIN"] = "local" + env["ENABLE_HOST_NAT"] = "true" if provider_net.enable_host_nat else "false" else: logger.warning( f"No provider network found for VDC {vdc_id} using network_id={network_id}; " diff --git a/app/utils/nscontroller_helpers.py b/app/utils/nscontroller_helpers.py index 139359f..1c3ad44 100644 --- a/app/utils/nscontroller_helpers.py +++ b/app/utils/nscontroller_helpers.py @@ -56,6 +56,7 @@ def dispatch_nscontroller_to_host(nsc_workload: Workload, network: Network, host "env": nsc_config.get("env", {}), "cpu": nsc_config.get("cpu", 4), "mem_limit": nsc_config.get("mem_limit", 128), + "enable_host_nat": nsc_config.get("enable_host_nat", False), } if "network_ports" in nsc_config: nsc_container["network_ports"] = nsc_config["network_ports"] diff --git a/migrations/versions/0003_network_enable_host_nat.py b/migrations/versions/0003_network_enable_host_nat.py new file mode 100644 index 0000000..8cf8ebc --- /dev/null +++ b/migrations/versions/0003_network_enable_host_nat.py @@ -0,0 +1,23 @@ +"""Add enable_host_nat to networks for NSController internet egress via NAT + +Revision ID: 0003_network_enable_host_nat +Revises: 0002_workload_excluded_hosts +Create Date: 2026-08-08 +""" +from alembic import op +import sqlalchemy as sa + +revision = '0003_network_enable_host_nat' +down_revision = '0002_workload_excluded_hosts' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table('networks', schema=None) as batch_op: + batch_op.add_column(sa.Column('enable_host_nat', sa.Boolean(), nullable=False, server_default='0')) + + +def downgrade(): + with op.batch_alter_table('networks', schema=None) as batch_op: + batch_op.drop_column('enable_host_nat') diff --git a/sandbox/containers/nscontroller/dnsmasq.conf b/sandbox/containers/nscontroller/dnsmasq.conf index 603919f..403fbcc 100644 --- a/sandbox/containers/nscontroller/dnsmasq.conf +++ b/sandbox/containers/nscontroller/dnsmasq.conf @@ -4,6 +4,12 @@ # Use bind-dynamic to support late-added interfaces (OVS ports) bind-dynamic +# eth0 is the Docker-bridge uplink used for host NAT (enable_host_nat). +# bind-dynamic binds every interface by default, so +# prevent dnsmasq to answer DNS/DHCP queries there! +except-interface=eth0 +no-dhcp-interface=eth0 + # Upstream DNS servers (overridable via UPSTREAM_DNS env var in entrypoint) server=8.8.8.8 server=1.1.1.1 diff --git a/sandbox/containers/nscontroller/entrypoint.sh b/sandbox/containers/nscontroller/entrypoint.sh index 48df264..c09d0d6 100755 --- a/sandbox/containers/nscontroller/entrypoint.sh +++ b/sandbox/containers/nscontroller/entrypoint.sh @@ -72,6 +72,23 @@ else echo "PROVIDER_CIDR / DHCP range not set – skipping provider DHCP config" fi +# Enabled per-network via the enable_host_nat flag. +# That gives this ns eth0 with a Docker-managed route; +if [ "$ENABLE_HOST_NAT" = "true" ] && [ -n "$PROVIDER_CIDR" ]; then + echo "Enabling host NAT: ${PROVIDER_CIDR} -> eth0" + + if ! iptables -t nat -C POSTROUTING -s "$PROVIDER_CIDR" -o eth0 -j MASQUERADE 2>/dev/null; then + iptables -t nat -A POSTROUTING -s "$PROVIDER_CIDR" -o eth0 -j MASQUERADE + fi + + # TODO(egress-security): FORWARD defaults to ACCEPT, so tenant VMs can + # currently reach anything routable from eth0 (host LAN, other tenants' + # NAT peers, etc). Add a default-DROP FORWARD policy + explicit allow + # rules before relying on this outside of trusted/dev environments. +else + echo "Host NAT disabled (ENABLE_HOST_NAT=${ENABLE_HOST_NAT:-false})" +fi + # Log configuration echo "DNS Configuration:" echo " - dnsmasq version: $(dnsmasq --version | head -n1)" diff --git a/worker/worker_tasks/container.py b/worker/worker_tasks/container.py index 2af647a..138d80e 100644 --- a/worker/worker_tasks/container.py +++ b/worker/worker_tasks/container.py @@ -16,11 +16,12 @@ from worker_tasks.ovs_sdn import OVS_SDN # Import DNS configuration constants try: - from app.utils.constants import GLOBAL_DNS_CONFIG_BASE_PATH, DNS_CONTAINER_MOUNT_PATH + from app.utils.constants import GLOBAL_DNS_CONFIG_BASE_PATH, DNS_CONTAINER_MOUNT_PATH, NSCONTROLLER_NAT_NETWORK except ImportError: # Fallback if app.utils.constants is not available in worker environment GLOBAL_DNS_CONFIG_BASE_PATH = "/var/lib/xcloudify/dns-configs" DNS_CONTAINER_MOUNT_PATH = "/etc/dnsmasq.d" + NSCONTROLLER_NAT_NETWORK = "xcloudify-wan" def _should_use_sudo() -> bool: @@ -657,8 +658,16 @@ class ContainerTask: network_ports = container_spec.get("network_ports", []) has_ovs_bridge = any(p.get("ovs_bridge") for p in network_ports) + enable_host_nat = bool(container_spec.get("enable_host_nat", False)) + if is_nscontroller: - if has_ovs_bridge: + if enable_host_nat: + # Give the NSController a uplink (eth0 + default route + + # Docker's automatic MASQUERADE) so its entrypoint can NAT the + # tenant subnet to the internet. + network_mode = self._ensure_nat_network() + self.logger.info(f"NSController: using network_mode={network_mode} (host NAT enabled)") + elif has_ovs_bridge: # Use OVS networking - container gets no Docker network, OVS ports attached later network_mode = "none" self.logger.info("NSController: using network_mode=none (OVS ports will be added)") @@ -687,6 +696,8 @@ class ContainerTask: if is_nscontroller: container_kwargs["cap_add"] = ["NET_ADMIN", "NET_RAW"] self.logger.info(f"NSController {system_container_id}: adding NET_ADMIN + NET_RAW capabilities") + if enable_host_nat: + container_kwargs["sysctls"] = {"net.ipv4.ip_forward": "1"} # Add restart policy from container spec if provided, otherwise use default from settings restart_policy = container_spec.get("restart_policy", settings.get_value("RESTART_POLICY", "always")) @@ -1331,12 +1342,18 @@ class ContainerTask: return False # ─────────────────── 2b️⃣ NETWORK MODE (NSController only) ─────────────────── - # NSControllers should use network_mode="none" if OVS ports will be added, - # otherwise use "bridge" for Docker port publishing (matching launch_container logic) + # NSControllers use the NAT uplink network if host NAT is enabled, + # else network_mode="none" if OVS ports will be added, else "bridge" + # for Docker port publishing (matching launch_container logic) if container_spec.get("workload_type") == "NSController": network_ports = container_spec.get("network_ports", []) has_ovs_bridge = any(p.get("ovs_bridge") for p in network_ports) - expected_network_mode = "none" if has_ovs_bridge else "bridge" + if container_spec.get("enable_host_nat"): + expected_network_mode = NSCONTROLLER_NAT_NETWORK + elif has_ovs_bridge: + expected_network_mode = "none" + else: + expected_network_mode = "bridge" running_network_mode = host_config.get("NetworkMode", "") if running_network_mode != expected_network_mode: self.logger.warning( @@ -1803,6 +1820,44 @@ class ContainerTask: except Exception: pass + def _ensure_nat_network(self) -> str: + """ + Ensure the NAT uplink docker bridge network exists on this host. + + NSControllers launched here (instead of network_mode="none"/"bridge") + get a real eth0 + default route + Docker's automatic MASQUERADE for + that subnet, which their entrypoint uses to NAT the tenant network's + egress traffic out to the internet. + """ + try: + self.docker_client.networks.get(NSCONTROLLER_NAT_NETWORK) + except docker.errors.NotFound: + self.logger.info(f"Creating NAT uplink network '{NSCONTROLLER_NAT_NETWORK}'") + try: + self.docker_client.networks.create( + NSCONTROLLER_NAT_NETWORK, + driver="bridge", + options={ + "com.docker.network.bridge.name": NSCONTROLLER_NAT_NETWORK, + # NSControllers for different tenants share this uplink network; + # they shouldn't be able to reach each other directly through it. + "com.docker.network.bridge.enable_icc": "false", + }, + ) + except docker.errors.APIError as e: + # Another concurrent launch on this host may have just created it. + if not self._docker_network_exists(NSCONTROLLER_NAT_NETWORK): + raise + self.logger.debug(f"NAT uplink network '{NSCONTROLLER_NAT_NETWORK}' created concurrently: {e}") + return NSCONTROLLER_NAT_NETWORK + + def _docker_network_exists(self, network_name: str) -> bool: + try: + self.docker_client.networks.get(network_name) + return True + except docker.errors.NotFound: + return False + def attach_to_network(self, container, network_name): """Attach an existing container to a Docker network.""" try: