JamesBhattarai/3cloud-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Port forwarding improvements inc multiple ports per container inc streamlit

Browse Source
This commit is contained in:
nexgen_mirrors
2025-05-20 16:53:35 +09:30
parent 41773f64ee
commit bb8bd7e80a
2 changed files with 196 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/controller/api/workload_container_routes.py
+93 -40
View File
@@ -15,6 +15,7 @@ from sqlalchemy import or_
websocket_server_url = "http://127.0.0.1:6000/api/create_task"
def validate_payload(payload):
"""
Validate the input payload according to the specified rules and return a sanitized version.
@@ -52,13 +53,6 @@ def validate_payload(payload):
'containers': []
}
#Confirm if the use_dns flag was set on the request at pod lvel or not
if 'use_dns' in payload:
sanitized_payload['use_dns']=False
if str(payload['use_dns']).lower=="true" or payload['use_dns']==True:
sanitized_payload['use_dns']=True
# Validate each container in the 'containers' list and build the sanitized version
for container in payload['containers']:
if not isinstance(container, dict):
@@ -100,10 +94,17 @@ def validate_payload(payload):
raise ValueError("Each port mapping must have 'internal' and 'external' keys.")
if not isinstance(port_mapping['internal'], int) or not isinstance(port_mapping['external'], int):
raise ValueError("'internal' and 'external' ports must be integers.")
sanitized_ports.append({
# Handle use_dns in port mapping
port_data = {
'internal': port_mapping['internal'],
'external': port_mapping['external']
})
}
if 'use_dns' in port_mapping:
port_data['use_dns'] = bool(port_mapping['use_dns'])
sanitized_ports.append(port_data)
if sanitized_ports:
sanitized_container['ports'] = sanitized_ports
@@ -126,12 +127,12 @@ def validate_payload(payload):
else:
raise ValueError("'networks' must be a string or a list of strings.")
# Add the sanitized container to the sanitized payload
sanitized_payload['containers'].append(sanitized_container)
return sanitized_payload
def find_available_external_port(host_id, preferred_port=None, port_range=(30000, 40000)):
"""Find a free external port on a given host."""
existing_ports = set(
@@ -214,6 +215,10 @@ def add_container_workload():
client_response = []
# Track Cloudflare configurations for bulk setup
ingress_mappings = []
dns_records_to_create = []
# --- Step 3: Create all Container Workloads inside the Pod ---
for _container in validated_data['containers']:
logger.debug(f"Creating container {_container['container_name']}")
@@ -241,6 +246,7 @@ def add_container_workload():
for port_mapping in _container['ports']:
internal_port = port_mapping['internal']
preferred_external_port = port_mapping['external']
use_dns = port_mapping.get('use_dns', False)
assigned_external_port = find_available_external_port(
selected_host.id, preferred_port=preferred_external_port
@@ -265,15 +271,29 @@ def add_container_workload():
# Save the pod-level forwarding rule
new_forward = PortForwarding(
#TODO - Remove the need to set a fake name, OR set a real name
name="s",
name="f",
pod_id=new_pod.id,
internal_port=internal_port,
external_port=assigned_external_port,
protocol=port_mapping.get("protocol", "tcp"),
ip_address=selected_host.ip_address_northsouth
)
db.session.add(new_forward)
db.session.add(new_forward)
db.session.flush() # Ensure we have new_forward.id
# If use_dns is True for this port, prepare Cloudflare config
if use_dns:
hostname = f"{_container['container_name']}-{internal_port}.hawkvelt.tech"
ingress_mappings.append({
"dns_hostname": hostname,
"local_ip": "127.0.0.1",
"local_port": str(internal_port)
})
dns_records_to_create.append({
"hostname": hostname,
"forward_id": new_forward.id,
"internal_port": internal_port
})
client_response.append({
"pod_id": str(new_pod.id),
@@ -282,44 +302,76 @@ def add_container_workload():
"container_status": "pending-allocation",
})
if validated_data['use_dns']:
logger.debug("use_dns set to true, building Cloudflare tunnel")
# Handle Cloudflare setup if we have any DNS-enabled ports
if ingress_mappings:
logger.debug(f"Setting up Cloudflare tunnel with {len(ingress_mappings)} routes")
api_token = "ri6lIjM-aRJBY_xZ82w0Haew93U6YgZYHi5jby1-"
account_id = "5095a74b62fee53cc5d997c67443bac5"
zone_id = "e2cafdd8929869d5db885d8824f514b5"
logger.info("Adding Cloudflare")
cloudflare_manager = CloudflareTunnelManager(api_token, account_id, zone_id, logger)
cloudflare_response = cloudflare_manager.setup_tunnel(
tunnel_name=f"tun_{new_container.id}",
dns_hostname=f"{new_container.id}.hawkvelt.tech",
local_ip_port=f"127.0.0.1:{_container['ports'][0]['internal']}")
logger.debug(cloudflare_response)
cloudflare_tunnel_token=cloudflare_response['token']
new_cloudflare_tunnel=CloudflareTunnel()
new_cloudflare_tunnel.token=cloudflare_tunnel_token
new_cloudflare_tunnel.account_id=account_id
new_cloudflare_tunnel.tunnel_id=cloudflare_response['tunnel_id']
new_cloudflare_tunnel.name=cloudflare_response['tunnel_name']
new_cloudflare_tunnel.tunnel_secret=cloudflare_response['tunnel_secret']
new_cloudflare_tunnel.nscontroller_workload_id=new_nscontroller.id
cloudflare_manager = CloudflareTunnelManager(api_token, account_id, zone_id, logger)
# Create a single tunnel with multiple ingress rules
tunnel_name = f"tun-{new_pod.id}"
cloudflare_response = cloudflare_manager.setup_tunnel(
tunnel_name=tunnel_name,
ingress_mappings=ingress_mappings
)
logger.debug(cloudflare_response)
cloudflare_tunnel_token = cloudflare_response['token']
# Create tunnel record in our database
new_cloudflare_tunnel = CloudflareTunnel(
token=cloudflare_tunnel_token,
account_id=account_id,
tunnel_id=cloudflare_response['tunnel_id'],
name=cloudflare_response['tunnel_name'],
tunnel_secret=cloudflare_response['tunnel_secret'],
nscontroller_workload_id=new_nscontroller.id
)
db.session.add(new_cloudflare_tunnel)
db.session.flush()
logger.info("Adding Cloudflare complete")
sidecar_container_name = f"cloudflared-sidecar-{new_container.id}"
# Create DNS records for each hostname
for dns_record in dns_records_to_create:
# Create DNS record in Cloudflare
# dns_response = cloudflare_manager.create_dns_record(
# hostname=dns_record['hostname'],
# target=f"{cloudflare_response['tunnel_id']}.cfargotunnel.com"
# )
# TODO - Make sure we are connecting the DNS record in the DB t whjats in Cloudflare so we can delete them later
# The commented code above is not required becuase the DNS records are created with the tunnel, so we need to query
# the setup_tunnel reponse for the DNs info...maybe?
# Create DNS record in our database
new_dns_record = CloudflareDNSRecord(
name="dns",
zone_id=zone_id,
dns_record_id="11", #dns_response['id'],
hostname=dns_record['hostname'],
record_type="CNAME",
content=f"{cloudflare_response['tunnel_id']}.cfargotunnel.com",
tunnel_id=new_cloudflare_tunnel.id
)
db.session.add(new_dns_record)
# Link the port forwarding to the DNS record
port_forward = PortForwarding.query.get(dns_record['forward_id'])
if port_forward:
port_forward.dns_record_id = new_dns_record.id
db.session.add(port_forward)
# Create cloudflared sidecar container
sidecar_container_name = f"cloudflared-sidecar-{new_pod.id}"
sidecar_launch_params = {
"docker_image": "cloudflare/cloudflared:latest",
"container_name": sidecar_container_name,
"command": f"tunnel --no-autoupdate run --token {cloudflare_tunnel_token}",
"network_mode": f"container:{new_nscontroller.name}", # Attach to NSController network namespace
"network_mode": f"container:{new_nscontroller.name}",
"restart_policy": "always"
}
# Create sidecar workload
sidecar_workload = Workload(
name=sidecar_container_name,
workload_type="Container",
@@ -330,7 +382,6 @@ def add_container_workload():
db.session.add(sidecar_workload)
db.session.flush()
# Map sidecar into the same pod
sidecar_mapping = ContainerPodContainer(
pod_id=new_pod.id,
container_workload_id=sidecar_workload.id
@@ -342,9 +393,11 @@ def add_container_workload():
"container_id": str(sidecar_workload.id),
"container_name": sidecar_workload.name,
"container_status": "pending-allocation",
"cloudflare_tunnel": {
"tunnel_id": new_cloudflare_tunnel.tunnel_id,
"hostnames": [dns['hostname'] for dns in dns_records_to_create]
}
})
else:
logger.debug("use_dns not True, skipping Cloudflare")
# --- Step 5: Update NSController launch params with final ports ---
new_nscontroller.launch_params = json.dumps(nscontroller_launch_params)
streamlit_server/views/containers.py
+103 -14
View File
@@ -26,6 +26,7 @@ def render_list():
# Create Container Workload Form
with st.expander("Create New Container Workload"):
# The form only contains form-specific widgets
with st.form("create_container"):
# Dropdown for VDC selection
selected_vdc_name_create = st.selectbox("VDC", list(vdc_options.keys()))
@@ -40,18 +41,65 @@ def render_list():
index=0
)
# If 'Custom...' selected, let user input their own image
# If 'Custom...' is selected, let the user input their own image
if docker_image == "Custom...":
docker_image = st.text_input("Custom Docker Image", value="")
# Port mapping (single)
st.markdown("**Optional: Port Mapping**")
internal_port = st.number_input("Internal Port", min_value=1, max_value=65535, step=1)
external_port = st.number_input("External Port", min_value=1, max_value=65535, step=1)
# Port mappings (multiple)
st.markdown("**Port Mappings**")
# Initialize port mappings in session state if not exists
if 'port_mappings' not in st.session_state:
st.session_state.port_mappings = [{
'internal': None,
'external': None,
'use_dns': True,
'protocol': 'tcp'
}]
# New: Use DNS (Cloudflare) Checkbox
use_dns = st.checkbox("Use DNS (Cloudflare)", value=True)
# Display each port mapping
for i, mapping in enumerate(st.session_state.port_mappings):
with st.container():
cols = st.columns([2, 2, 2, 1, 1])
with cols[0]:
internal_port = st.number_input(
"Internal Port",
min_value=1,
max_value=65535,
step=1,
value=mapping['internal'],
key=f"internal_{i}"
)
with cols[1]:
external_port = st.number_input(
"External Port",
min_value=1,
max_value=65535,
step=1,
value=mapping['external'],
key=f"external_{i}"
)
with cols[2]:
protocol = st.selectbox(
"Protocol",
["tcp", "udp"],
index=0 if mapping['protocol'] == 'tcp' else 1,
key=f"protocol_{i}"
)
with cols[3]:
use_dns = st.checkbox(
"Use DNS",
value=mapping['use_dns'],
key=f"use_dns_{i}"
)
# with cols[4]:
# # Only show remove button for additional port mappings
# if i > 0:
# if st.button("❌", key=f"remove_{i}"):
# st.session_state.port_mappings.pop(i)
# st.rerun()
# Form submit button for creating the container
submit = st.form_submit_button("Create Container Workload")
if submit and selected_vdc_name_create and container_name:
@@ -63,20 +111,51 @@ def render_list():
"networks": "bridge", # Default network
}
# Only add ports if both fields are non-zero
if internal_port and external_port:
container_payload["ports"] = [{"internal": int(internal_port), "external": int(external_port)}]
# Prepare port mappings from session state
port_mappings = []
for i in range(len(st.session_state.port_mappings)):
internal = st.session_state.get(f"internal_{i}")
external = st.session_state.get(f"external_{i}")
protocol = st.session_state.get(f"protocol_{i}")
use_dns = st.session_state.get(f"use_dns_{i}")
if internal and external: # Only add if both ports are specified
port_mappings.append({
"internal": int(internal),
"external": int(external),
"protocol": protocol,
"use_dns": use_dns
})
if port_mappings:
container_payload["ports"] = port_mappings
payload = {
"vdc": vdc_id,
"use_dns": use_dns,
"containers": [container_payload]
}
result = st.session_state.client.create_container(payload)
if result:
st.success("Container workload created successfully!")
# Clear port mappings after successful creation
st.session_state.port_mappings = [{
'internal': None,
'external': None,
'use_dns': True,
'protocol': 'tcp'
}]
st.rerun()
# The button to add a new port mapping is now placed outside the form
if st.button("➕ Add Another Port Mapping"):
st.session_state.port_mappings.append({
'internal': None,
'external': None,
'use_dns': True,
'protocol': 'tcp'
})
st.rerun()
# List Container Pods
st.subheader("Existing Container Pods")
@@ -128,11 +207,21 @@ def render_list():
st.markdown("🔁 **Port Forwarding:**")
for pf in pod['port_forwardings']:
print (pf)
st.text(f" {pf['protocol'].upper()} → {pf['external_port']} ↦ {pf['internal_port']}")
dns_status = "🌐 DNS" if pf.get('dns_record_id') else "🚫 No DNS"
cols = st.columns([1, 3, 2])
with cols[0]:
st.markdown(f"**{pf['protocol'].upper()}**")
with cols[1]:
st.text(f"{pf['external_port']} → {pf['internal_port']}")
with cols[2]:
st.text(dns_status)
st.page_link(f"http://{pf['ip_address']}:{pf['external_port']}",label=f"http://{pf['ip_address']}:{pf['external_port']}")
if pf.get('dns_hostname'):
st.page_link(f"http://{pf['dns_hostname']}", label=f"http://{pf['dns_hostname']}")
else:
st.page_link(f"http://{pf['ip_address']}:{pf['external_port']}", label=f"http://{pf['ip_address']}:{pf['external_port']}")
st.markdown("---")
else:
st.text("No port forwardings")