JamesBhattarai/3cloud-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: Implement NAT for Private network

Browse Source 
Added enable_nat bool for networks to allow nats
Implemented docker network bridge, to allow nat with tenancy seperated as `enable_icc:false`
This commit is contained in:
JamesBhattarai
2026-08-08 17:41:15 +02:00
parent be0d63f5dd
commit 7ec2e4d47b
11 changed files with 131 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/controller/api/network_routes.py
+2
View File
@@ -61,6 +61,7 @@ def validate_network_data(data, is_update=False):
'ipv6_cidr': str,
'ipv6_gateway': str,
'ovs_bridge': str,
'enable_host_nat': bool,
'encapsulation': str,
}
for field, field_type in optional_fields.items():
@@ -139,6 +140,7 @@ def add_network():
ipv6_cidr=validated_data.get('ipv6_cidr'),
ipv6_gateway=validated_data.get('ipv6_gateway'),
ovs_bridge=validated_data.get('ovs_bridge'),
enable_host_nat=validated_data.get('enable_host_nat', False),
**({'encapsulation': validated_data['encapsulation']} if 'encapsulation' in validated_data else {}),
)
db.session.add(instance)
app/controller/api/workload_host_routes.py
+1
View File
@@ -784,6 +784,7 @@ def get_container_workloads_for_host(workload_host_id):
"mem_limit": nsc_config.get("mem_limit", 128),
"pod_id": f"standalone-nsc-{nsc.id}",
"network_ports": [], # Initialize as empty; will populate if port exists
"enable_host_nat": nsc_config.get("enable_host_nat", False),
}
if port:
app/models/network.py
+1
View File
@@ -78,6 +78,7 @@ class Network(BaseModel):
encapsulation = Column(String(50), default="VXLAN", nullable=False)
ovs_bridge = Column(String(255), nullable=True)
network_type = Column(String(255), nullable=True)
enable_host_nat = Column(Boolean, default=False, nullable=False, server_default="0")
def __init__(self, *args, **kwargs):
# Convert UUID strings to UUID objects for any UUID fields
app/utils/build_pod_payload.py
+13 -10
View File
@@ -91,17 +91,20 @@ def build_pod_payload(pod: ContainerPod, deleted_container_id: Optional[str] = N
}
# Include ports for NSController
if _container.workload_type == "NSController" and "ports" in launch_params:
cont["ports"] = launch_params["ports"]
if _container.workload_type == "NSController":
cont["enable_host_nat"] = launch_params.get("enable_host_nat", False)
if "ports" in launch_params:
cont["ports"] = launch_params["ports"]
# Include DNS configuration fields for NSController
if "use_dns" in launch_params:
cont["use_dns"] = launch_params["use_dns"]
if "vdc_id" in launch_params:
cont["vdc_id"] = launch_params["vdc_id"]
if "dns_config" in launch_params:
cont["dns_config"] = launch_params["dns_config"]
# Include DNS configuration fields for NSController
if "use_dns" in launch_params:
cont["use_dns"] = launch_params["use_dns"]
if "vdc_id" in launch_params:
cont["vdc_id"] = launch_params["vdc_id"]
if "dns_config" in launch_params:
cont["dns_config"] = launch_params["dns_config"]
# Check if this is a special lifecycle operation (restart)
if _container.status == "pending-restart":
cont["restart"] = True
app/utils/constants.py
+4
View File
@@ -28,3 +28,7 @@ CLOUDFLARED_IMAGE = "cloudflare/cloudflared:latest"
# Do not hard-code any fallback here; if not set, payload builders must omit
# the ovs_bridge key when a network does not define it.
DEFAULT_OVS_BRIDGE = os.getenv('XCF_DEFAULT_OVS_BRIDGE')
# Docker bridge network NSControllers attach to when a network has
# enable_host_nat set, for internet egress.
NSCONTROLLER_NAT_NETWORK = "xcloudify-wan"
app/utils/nscontroller_config.py
+3
View File
@@ -111,6 +111,8 @@ def build_nscontroller_config(
"network_id": network_id, # Worker/proxy uses this to scope metadata lookups
"dns_config": dns_config, # Initial DNS configuration payload
"env": env, # Dynamic provider network environment variables
# To allow nat via Docker-bridge uplink
"enable_host_nat": bool(provider_net.enable_host_nat),
}
# Dynamic ports from database (for runtime operations)
@@ -202,6 +204,7 @@ def _build_env(vdc_id: str, network_id: str) -> Dict[str, str]:
env["PROVIDER_DHCP_RANGE_START"] = provider_net.dhcp_range_start or ""
env["PROVIDER_DHCP_RANGE_END"] = provider_net.dhcp_range_end or ""
env["PROVIDER_DOMAIN"] = "local"
env["ENABLE_HOST_NAT"] = "true" if provider_net.enable_host_nat else "false"
else:
logger.warning(
f"No provider network found for VDC {vdc_id} using network_id={network_id}; "
app/utils/nscontroller_helpers.py
+1
View File
@@ -56,6 +56,7 @@ def dispatch_nscontroller_to_host(nsc_workload: Workload, network: Network, host
"env": nsc_config.get("env", {}),
"cpu": nsc_config.get("cpu", 4),
"mem_limit": nsc_config.get("mem_limit", 128),
"enable_host_nat": nsc_config.get("enable_host_nat", False),
}
if "network_ports" in nsc_config:
nsc_container["network_ports"] = nsc_config["network_ports"]
migrations/versions/0003_network_enable_host_nat.py
+23
View File
@@ -0,0 +1,23 @@
"""Add enable_host_nat to networks for NSController internet egress via NAT
Revision ID: 0003_network_enable_host_nat
Revises: 0002_workload_excluded_hosts
Create Date: 2026-08-08
"""
from alembic import op
import sqlalchemy as sa
revision = '0003_network_enable_host_nat'
down_revision = '0002_workload_excluded_hosts'
branch_labels = None
depends_on = None
def upgrade():
with op.batch_alter_table('networks', schema=None) as batch_op:
batch_op.add_column(sa.Column('enable_host_nat', sa.Boolean(), nullable=False, server_default='0'))
def downgrade():
with op.batch_alter_table('networks', schema=None) as batch_op:
batch_op.drop_column('enable_host_nat')
sandbox/containers/nscontroller/dnsmasq.conf
+6
View File
@@ -4,6 +4,12 @@
# Use bind-dynamic to support late-added interfaces (OVS ports)
bind-dynamic
# eth0 is the Docker-bridge uplink used for host NAT (enable_host_nat).
# bind-dynamic binds every interface by default, so
# prevent dnsmasq to answer DNS/DHCP queries there!
except-interface=eth0
no-dhcp-interface=eth0
# Upstream DNS servers (overridable via UPSTREAM_DNS env var in entrypoint)
server=8.8.8.8
server=1.1.1.1
sandbox/containers/nscontroller/entrypoint.sh
+17
View File
@@ -72,6 +72,23 @@ else
echo "PROVIDER_CIDR / DHCP range not set – skipping provider DHCP config"
fi
# Enabled per-network via the enable_host_nat flag.
# That gives this ns eth0 with a Docker-managed route;
if [ "$ENABLE_HOST_NAT" = "true" ] && [ -n "$PROVIDER_CIDR" ]; then
echo "Enabling host NAT: ${PROVIDER_CIDR} -> eth0"
if ! iptables -t nat -C POSTROUTING -s "$PROVIDER_CIDR" -o eth0 -j MASQUERADE 2>/dev/null; then
iptables -t nat -A POSTROUTING -s "$PROVIDER_CIDR" -o eth0 -j MASQUERADE
fi
# TODO(egress-security): FORWARD defaults to ACCEPT, so tenant VMs can
# currently reach anything routable from eth0 (host LAN, other tenants'
# NAT peers, etc). Add a default-DROP FORWARD policy + explicit allow
# rules before relying on this outside of trusted/dev environments.
else
echo "Host NAT disabled (ENABLE_HOST_NAT=${ENABLE_HOST_NAT:-false})"
fi
# Log configuration
echo "DNS Configuration:"
echo " - dnsmasq version: $(dnsmasq --version | head -n1)"
worker/worker_tasks/container.py
+60 -5
View File
@@ -16,11 +16,12 @@ from worker_tasks.ovs_sdn import OVS_SDN
# Import DNS configuration constants
try:
from app.utils.constants import GLOBAL_DNS_CONFIG_BASE_PATH, DNS_CONTAINER_MOUNT_PATH
from app.utils.constants import GLOBAL_DNS_CONFIG_BASE_PATH, DNS_CONTAINER_MOUNT_PATH, NSCONTROLLER_NAT_NETWORK
except ImportError:
# Fallback if app.utils.constants is not available in worker environment
GLOBAL_DNS_CONFIG_BASE_PATH = "/var/lib/xcloudify/dns-configs"
DNS_CONTAINER_MOUNT_PATH = "/etc/dnsmasq.d"
NSCONTROLLER_NAT_NETWORK = "xcloudify-wan"
def _should_use_sudo() -> bool:
@@ -657,8 +658,16 @@ class ContainerTask:
network_ports = container_spec.get("network_ports", [])
has_ovs_bridge = any(p.get("ovs_bridge") for p in network_ports)
enable_host_nat = bool(container_spec.get("enable_host_nat", False))
if is_nscontroller:
if has_ovs_bridge:
if enable_host_nat:
# Give the NSController a uplink (eth0 + default route +
# Docker's automatic MASQUERADE) so its entrypoint can NAT the
# tenant subnet to the internet.
network_mode = self._ensure_nat_network()
self.logger.info(f"NSController: using network_mode={network_mode} (host NAT enabled)")
elif has_ovs_bridge:
# Use OVS networking - container gets no Docker network, OVS ports attached later
network_mode = "none"
self.logger.info("NSController: using network_mode=none (OVS ports will be added)")
@@ -687,6 +696,8 @@ class ContainerTask:
if is_nscontroller:
container_kwargs["cap_add"] = ["NET_ADMIN", "NET_RAW"]
self.logger.info(f"NSController {system_container_id}: adding NET_ADMIN + NET_RAW capabilities")
if enable_host_nat:
container_kwargs["sysctls"] = {"net.ipv4.ip_forward": "1"}
# Add restart policy from container spec if provided, otherwise use default from settings
restart_policy = container_spec.get("restart_policy", settings.get_value("RESTART_POLICY", "always"))
@@ -1331,12 +1342,18 @@ class ContainerTask:
return False
# ─────────────────── 2b️⃣ NETWORK MODE (NSController only) ───────────────────
# NSControllers should use network_mode="none" if OVS ports will be added,
# otherwise use "bridge" for Docker port publishing (matching launch_container logic)
# NSControllers use the NAT uplink network if host NAT is enabled,
# else network_mode="none" if OVS ports will be added, else "bridge"
# for Docker port publishing (matching launch_container logic)
if container_spec.get("workload_type") == "NSController":
network_ports = container_spec.get("network_ports", [])
has_ovs_bridge = any(p.get("ovs_bridge") for p in network_ports)
expected_network_mode = "none" if has_ovs_bridge else "bridge"
if container_spec.get("enable_host_nat"):
expected_network_mode = NSCONTROLLER_NAT_NETWORK
elif has_ovs_bridge:
expected_network_mode = "none"
else:
expected_network_mode = "bridge"
running_network_mode = host_config.get("NetworkMode", "")
if running_network_mode != expected_network_mode:
self.logger.warning(
@@ -1803,6 +1820,44 @@ class ContainerTask:
except Exception:
pass
def _ensure_nat_network(self) -> str:
"""
Ensure the NAT uplink docker bridge network exists on this host.
NSControllers launched here (instead of network_mode="none"/"bridge")
get a real eth0 + default route + Docker's automatic MASQUERADE for
that subnet, which their entrypoint uses to NAT the tenant network's
egress traffic out to the internet.
"""
try:
self.docker_client.networks.get(NSCONTROLLER_NAT_NETWORK)
except docker.errors.NotFound:
self.logger.info(f"Creating NAT uplink network '{NSCONTROLLER_NAT_NETWORK}'")
try:
self.docker_client.networks.create(
NSCONTROLLER_NAT_NETWORK,
driver="bridge",
options={
"com.docker.network.bridge.name": NSCONTROLLER_NAT_NETWORK,
# NSControllers for different tenants share this uplink network;
# they shouldn't be able to reach each other directly through it.
"com.docker.network.bridge.enable_icc": "false",
},
)
except docker.errors.APIError as e:
# Another concurrent launch on this host may have just created it.
if not self._docker_network_exists(NSCONTROLLER_NAT_NETWORK):
raise
self.logger.debug(f"NAT uplink network '{NSCONTROLLER_NAT_NETWORK}' created concurrently: {e}")
return NSCONTROLLER_NAT_NETWORK
def _docker_network_exists(self, network_name: str) -> bool:
try:
self.docker_client.networks.get(network_name)
return True
except docker.errors.NotFound:
return False
def attach_to_network(self, container, network_name):
"""Attach an existing container to a Docker network."""
try: