This commit is contained in:
Maxime Van Hees
2025-08-27 16:03:32 +02:00
parent 773db2238d
commit 784f87db97
4 changed files with 183 additions and 48 deletions

View File

@@ -266,12 +266,12 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
if !converted_ok { if !converted_ok {
// Attempt 2: pipe via stdout into dd (avoids qemu-img destination locking semantics on some FS) // Attempt 2: pipe via stdout into dd (avoids qemu-img destination locking semantics on some FS)
let cmd2 = format!( let heredoc2 = format!(
"#!/bin/bash -euo pipefail\nqemu-img convert -O raw {} - | dd of={} bs=4M status=none", "bash -e -s <<'EOF'\nset -euo pipefail\nqemu-img convert -O raw {} - | dd of={} bs=4M status=none\nEOF\n",
shell_escape(&disk_to_use), shell_escape(&disk_to_use),
shell_escape(&dest) shell_escape(&dest)
); );
match sal_process::run(&cmd2).silent(true).die(false).execute() { match sal_process::run(&heredoc2).silent(true).die(false).execute() {
Ok(res) if res.success => { Ok(res) if res.success => {
converted_ok = true; converted_ok = true;
} }
@@ -408,6 +408,30 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
let dhcp_end = let dhcp_end =
std::env::var("HERO_VIRT_DHCP_END").unwrap_or_else(|_| "172.30.0.250".into()); std::env::var("HERO_VIRT_DHCP_END").unwrap_or_else(|_| "172.30.0.250".into());
// Optional IPv6 over Mycelium provisioning (bridge P::2/64 + RA) guarded by env and detection.
let ipv6_env = std::env::var("HERO_VIRT_IPV6_ENABLE").unwrap_or_else(|_| "".into());
let mut ipv6_enabled = ipv6_env.eq_ignore_ascii_case("1") || ipv6_env.eq_ignore_ascii_case("true");
let mycelium_if_cfg = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| "mycelium".into());
let mut ipv6_bridge_cidr: Option<String> = None;
let mut mycelium_if_opt: Option<String> = None;
let enable_auto = ipv6_env.is_empty(); // auto-enable if mycelium is detected and not explicitly disabled
if ipv6_enabled || enable_auto {
if let Ok(cidr) = std::env::var("HERO_VIRT_IPV6_BRIDGE_CIDR") {
// Explicit override for bridge IPv6 (e.g., "400:...::2/64")
ipv6_bridge_cidr = Some(cidr);
mycelium_if_opt = Some(mycelium_if_cfg.clone());
ipv6_enabled = true;
} else if let Ok((ifname, myc_addr)) = get_mycelium_ipv6_addr(&mycelium_if_cfg) {
// Derive P::2/64 from the mycelium node address
if let Ok((_pfx, router_cidr)) = derive_ipv6_prefix_from_mycelium(&myc_addr) {
ipv6_bridge_cidr = Some(router_cidr);
mycelium_if_opt = Some(ifname);
ipv6_enabled = true;
}
}
}
// Ensure host-side networking (requires root privileges / CAP_NET_ADMIN) // Ensure host-side networking (requires root privileges / CAP_NET_ADMIN)
ensure_host_net_prereq_dnsmasq_nftables( ensure_host_net_prereq_dnsmasq_nftables(
&bridge_name, &bridge_name,
@@ -415,6 +439,8 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
&subnet_cidr, &subnet_cidr,
&dhcp_start, &dhcp_start,
&dhcp_end, &dhcp_end,
ipv6_bridge_cidr.as_deref(),
mycelium_if_opt.as_deref(),
)?; )?;
// Ensure a TAP device for this VM and attach to the bridge // Ensure a TAP device for this VM and attach to the bridge
@@ -432,18 +458,15 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> {
} }
let args_str = shell_join(&parts); let args_str = shell_join(&parts);
let script = format!( // Execute via a bash heredoc to avoid any quoting pitfalls
"#!/bin/bash -e let heredoc = format!(
nohup {} > '{}' 2>&1 & "bash -e -s <<'EOF'\nnohup {} > '{}' 2>&1 &\necho $! > '{}'\nEOF\n",
echo $! > '{}'
",
args_str, args_str,
log_file, log_file,
vm_pid_path(id).to_string_lossy() vm_pid_path(id).to_string_lossy()
); );
// Execute command; this will background cloud-hypervisor and return
// Execute script; this will background cloud-hypervisor and return let result = sal_process::run(&heredoc).execute();
let result = sal_process::run(&script).execute();
match result { match result {
Ok(res) => { Ok(res) => {
if !res.success { if !res.success {
@@ -644,9 +667,8 @@ fn tap_name_for_id(id: &str) -> String {
fn ensure_tap_for_vm(bridge_name: &str, id: &str) -> Result<String, CloudHvError> { fn ensure_tap_for_vm(bridge_name: &str, id: &str) -> Result<String, CloudHvError> {
let tap = tap_name_for_id(id); let tap = tap_name_for_id(id);
let script = format!( let body = format!(
"#!/bin/bash -e "BR={br}
BR={br}
TAP={tap} TAP={tap}
UIDX=$(id -u) UIDX=$(id -u)
GIDX=$(id -g) GIDX=$(id -g)
@@ -661,8 +683,9 @@ ip link set \"$TAP\" up
br = shell_escape(bridge_name), br = shell_escape(bridge_name),
tap = shell_escape(&tap), tap = shell_escape(&tap),
); );
let heredoc_tap = format!("bash -e -s <<'EOF'\n{}\nEOF\n", body);
match sal_process::run(&script).silent(true).execute() { match sal_process::run(&heredoc_tap).silent(true).execute() {
Ok(res) if res.success => Ok(tap), Ok(res) if res.success => Ok(tap),
Ok(res) => Err(CloudHvError::CommandFailed(format!( Ok(res) => Err(CloudHvError::CommandFailed(format!(
"Failed to ensure TAP '{}': {}", "Failed to ensure TAP '{}': {}",
@@ -688,12 +711,75 @@ fn stable_mac_from_id(id: &str) -> String {
format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", b0, b1, b2, b3, b4, b5) format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", b0, b1, b2, b3, b4, b5)
} }
/// Discover the mycelium IPv6 address and validate the interface.
///
/// Returns (interface_name, address_string).
fn get_mycelium_ipv6_addr(iface_hint: &str) -> Result<(String, String), CloudHvError> {
// Parse `mycelium inspect` for "Address: ..."
let insp = sal_process::run("mycelium inspect").silent(true).die(false).execute();
let mut addr: Option<String> = None;
if let Ok(res) = insp {
if res.success {
for l in res.stdout.lines() {
let lt = l.trim();
if let Some(rest) = lt.strip_prefix("Address:") {
let val = rest.trim().trim_matches('"').to_string();
if !val.is_empty() {
addr = Some(val);
break;
}
}
}
}
}
let iface = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| iface_hint.to_string());
// Validate interface exists and has IPv6 configured (best-effort)
let cmd = format!("ip -6 addr show dev {}", shell_escape(&iface));
match sal_process::run(&cmd).silent(true).die(false).execute() {
Ok(r) if r.success => {
// proceed
}
_ => {
return Err(CloudHvError::DependencyMissing(format!(
"mycelium interface '{}' not found or no IPv6 configured",
iface
)));
}
}
if let Some(a) = addr {
Ok((iface, a))
} else {
Err(CloudHvError::DependencyMissing(
"failed to read mycelium IPv6 address via `mycelium inspect`".into(),
))
}
}
/// Derive a /64 prefix P from the mycelium IPv6 and return (P/64, P::2/64).
fn derive_ipv6_prefix_from_mycelium(m: &str) -> Result<(String, String), CloudHvError> {
let ip = m
.parse::<std::net::Ipv6Addr>()
.map_err(|e| CloudHvError::InvalidSpec(format!("invalid mycelium IPv6 address '{}': {}", m, e)))?;
let seg = ip.segments(); // [u16; 8]
// Take the top /64 from the mycelium address; zero the host half
let pfx = std::net::Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, 0);
// Router address for the bridge = P::2
let router = std::net::Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, 2);
let pfx_str = format!("{}/64", pfx);
let router_cidr = format!("{}/64", router);
Ok((pfx_str, router_cidr))
}
fn ensure_host_net_prereq_dnsmasq_nftables( fn ensure_host_net_prereq_dnsmasq_nftables(
bridge_name: &str, bridge_name: &str,
bridge_addr_cidr: &str, bridge_addr_cidr: &str,
subnet_cidr: &str, subnet_cidr: &str,
dhcp_start: &str, dhcp_start: &str,
dhcp_end: &str, dhcp_end: &str,
ipv6_bridge_cidr: Option<&str>,
mycelium_if: Option<&str>,
) -> Result<(), CloudHvError> { ) -> Result<(), CloudHvError> {
// Dependencies // Dependencies
for bin in ["ip", "nft", "dnsmasq", "systemctl"] { for bin in ["ip", "nft", "dnsmasq", "systemctl"] {
@@ -705,16 +791,18 @@ fn ensure_host_net_prereq_dnsmasq_nftables(
} }
} }
// Build idempotent setup script // Prepare optional IPv6 value (empty string when disabled)
let script = format!( let ipv6_cidr = ipv6_bridge_cidr.unwrap_or("");
"#!/bin/bash -e
set -e
// Build idempotent setup script
let body = format!(
"set -e
BR={br} BR={br}
BR_ADDR={br_addr} BR_ADDR={br_addr}
SUBNET={subnet} SUBNET={subnet}
DHCP_START={dstart} DHCP_START={dstart}
DHCP_END={dend} DHCP_END={dend}
IPV6_CIDR={v6cidr}
# Determine default WAN interface # Determine default WAN interface
WAN_IF=$(ip -o route show default | awk '{{print $5}}' | head -n1) WAN_IF=$(ip -o route show default | awk '{{print $5}}' | head -n1)
@@ -724,23 +812,27 @@ ip link show \"$BR\" >/dev/null 2>&1 || ip link add name \"$BR\" type bridge
ip addr replace \"$BR_ADDR\" dev \"$BR\" ip addr replace \"$BR_ADDR\" dev \"$BR\"
ip link set \"$BR\" up ip link set \"$BR\" up
# IPv6 placeholder address + forward (temporary) # IPv6 bridge address and forwarding (optional)
ip -6 addr add 400::1/64 dev \"$BR\" 2>/dev/null || true if [ -n \"$IPV6_CIDR\" ]; then
ip -6 addr replace \"$IPV6_CIDR\" dev \"$BR\"
sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null || true sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null || true
fi
# IPv4 forwarding # IPv4 forwarding
sysctl -w net.ipv4.ip_forward=1 >/dev/null sysctl -w net.ipv4.ip_forward=1 >/dev/null
# nftables NAT (idempotent) # nftables NAT (idempotent) for IPv4
nft list table ip hero >/dev/null 2>&1 || nft add table ip hero nft list table ip hero >/dev/null 2>&1 || nft add table ip hero
nft list chain ip hero postrouting >/dev/null 2>&1 || nft add chain ip hero postrouting {{ type nat hook postrouting priority 100 \\; }} nft list chain ip hero postrouting >/dev/null 2>&1 || nft add chain ip hero postrouting {{ type nat hook postrouting priority 100 \\; }}
nft list chain ip hero postrouting | grep -q \"ip saddr $SUBNET oifname \\\"$WAN_IF\\\" masquerade\" \ nft list chain ip hero postrouting | grep -q \"ip saddr $SUBNET oifname \\\"$WAN_IF\\\" masquerade\" \
|| nft add rule ip hero postrouting ip saddr $SUBNET oifname \"$WAN_IF\" masquerade || nft add rule ip hero postrouting ip saddr $SUBNET oifname \"$WAN_IF\" masquerade
# dnsmasq DHCP config (idempotent) # dnsmasq DHCPv4 + RA/DHCPv6 config (idempotent)
mkdir -p /etc/dnsmasq.d mkdir -p /etc/dnsmasq.d
CFG=/etc/dnsmasq.d/hero-$BR.conf CFG=/etc/dnsmasq.d/hero-$BR.conf
TMP=/etc/dnsmasq.d/.hero-$BR.conf.new TMP=/etc/dnsmasq.d/.hero-$BR.conf.new
# Always include IPv4 section
cat >\"$TMP\" <<EOF cat >\"$TMP\" <<EOF
interface=$BR interface=$BR
bind-interfaces bind-interfaces
@@ -748,6 +840,16 @@ dhcp-range=$DHCP_START,$DHCP_END,12h
dhcp-option=option:dns-server,1.1.1.1,8.8.8.8 dhcp-option=option:dns-server,1.1.1.1,8.8.8.8
EOF EOF
# Optionally append IPv6 RA/DHCPv6
if [ -n \"$IPV6_CIDR\" ]; then
cat >>\"$TMP\" <<'EOFV6'
enable-ra
dhcp-range=::,constructor:BR_PLACEHOLDER,ra-names,64,12h
dhcp-option=option6:dns-server,[2001:4860:4860::8888],[2606:4700:4700::1111]
EOFV6
sed -i \"s/BR_PLACEHOLDER/$BR/g\" \"$TMP\"
fi
if [ ! -f \"$CFG\" ] || ! cmp -s \"$CFG\" \"$TMP\"; then if [ ! -f \"$CFG\" ] || ! cmp -s \"$CFG\" \"$TMP\"; then
mv \"$TMP\" \"$CFG\" mv \"$TMP\" \"$CFG\"
if systemctl is-active --quiet dnsmasq; then if systemctl is-active --quiet dnsmasq; then
@@ -765,9 +867,13 @@ fi
subnet = shell_escape(subnet_cidr), subnet = shell_escape(subnet_cidr),
dstart = shell_escape(dhcp_start), dstart = shell_escape(dhcp_start),
dend = shell_escape(dhcp_end), dend = shell_escape(dhcp_end),
v6cidr = shell_escape(ipv6_cidr),
); );
match sal_process::run(&script).silent(true).execute() { // Use a unique heredoc delimiter to avoid clashing with inner <<EOF blocks
let heredoc_net = format!("bash -e -s <<'HERONET'\n{}\nHERONET\n", body);
match sal_process::run(&heredoc_net).silent(true).execute() {
Ok(res) if res.success => Ok(()), Ok(res) if res.success => Ok(()),
Ok(res) => Err(CloudHvError::CommandFailed(format!( Ok(res) => Err(CloudHvError::CommandFailed(format!(
"Host networking setup failed: {}", "Host networking setup failed: {}",

View File

@@ -141,6 +141,49 @@ pub fn host_check_deps() -> Result<HostCheckReport, HostCheckError> {
} }
} }
// Optional Mycelium IPv6 checks when enabled via env
let ipv6_env = std::env::var("HERO_VIRT_IPV6_ENABLE").unwrap_or_else(|_| "".into());
let ipv6_enabled = ipv6_env.eq_ignore_ascii_case("1") || ipv6_env.eq_ignore_ascii_case("true");
if ipv6_enabled {
// Require mycelium CLI
if bin_missing("mycelium") {
critical.push("mycelium CLI not found on PATH (required when HERO_VIRT_IPV6_ENABLE=true)".into());
}
// Validate interface presence and global IPv6
let ifname = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| "mycelium".into());
let check_if = sal_process::run(&format!("ip -6 addr show dev {}", ifname))
.silent(true)
.die(false)
.execute();
match check_if {
Ok(r) if r.success => {
let out = r.stdout;
if !(out.contains("inet6") && out.contains("scope global")) {
notes.push(format!(
"iface '{}' present but no global IPv6 detected; Mycelium may not be up yet",
ifname
));
}
}
_ => {
critical.push(format!(
"iface '{}' not found or no IPv6; ensure Mycelium is running",
ifname
));
}
}
// Best-effort: parse `mycelium inspect` for Address
let insp = sal_process::run("mycelium inspect").silent(true).die(false).execute();
match insp {
Ok(res) if res.success && res.stdout.contains("Address:") => {
// good enough
}
_ => {
notes.push("`mycelium inspect` did not return an Address; IPv6 overlay may be unavailable".into());
}
}
}
// Summarize ok flag // Summarize ok flag
let ok = critical.is_empty(); let ok = critical.is_empty();

View File

@@ -52,9 +52,9 @@ impl Default for NetPlanOpts {
fn default() -> Self { fn default() -> Self {
Self { Self {
dhcp4: true, dhcp4: true,
dhcp6: false, dhcp6: true,
ipv6_addr: Some("400::10/64".into()), ipv6_addr: None,
gw6: Some("400::1".into()), gw6: None,
} }
} }
} }
@@ -154,8 +154,6 @@ pub fn image_prepare(opts: &ImagePrepOptions) -> Result<ImagePrepResult, ImagePr
match opts.flavor { match opts.flavor {
Flavor::Ubuntu => { Flavor::Ubuntu => {
// Build bash script that performs all steps and echos "RAW|ROOT_UUID|BOOT_UUID" at end // Build bash script that performs all steps and echos "RAW|ROOT_UUID|BOOT_UUID" at end
let net_ipv6 = opts.net.ipv6_addr.clone().unwrap_or_else(|| "400::10/64".into());
let gw6 = opts.net.gw6.clone().unwrap_or_else(|| "400::1".into());
let disable_ci_net = opts.disable_cloud_init_net; let disable_ci_net = opts.disable_cloud_init_net;
// Keep script small and robust; avoid brace-heavy awk to simplify escaping. // Keep script small and robust; avoid brace-heavy awk to simplify escaping.
@@ -332,11 +330,6 @@ network:
eth0: eth0:
dhcp4: {dhcp4} dhcp4: {dhcp4}
dhcp6: {dhcp6} dhcp6: {dhcp6}
addresses:
- {ipv6}
routes:
- to: \"::/0\"
via: {gw6}
nameservers: nameservers:
addresses: [8.8.8.8, 1.1.1.1, 2001:4860:4860::8888] addresses: [8.8.8.8, 1.1.1.1, 2001:4860:4860::8888]
EOF EOF
@@ -370,8 +363,6 @@ exit 0
raw = shell_escape(&raw_path), raw = shell_escape(&raw_path),
dhcp4 = if opts.net.dhcp4 { "true" } else { "false" }, dhcp4 = if opts.net.dhcp4 { "true" } else { "false" },
dhcp6 = if opts.net.dhcp6 { "true" } else { "false" }, dhcp6 = if opts.net.dhcp6 { "true" } else { "false" },
ipv6 = shell_escape(&net_ipv6),
gw6 = shell_escape(&gw6),
disable_ci_net = if disable_ci_net { "true" } else { "false" }, disable_ci_net = if disable_ci_net { "true" } else { "false" },
); );

View File

@@ -17,7 +17,7 @@
// /images/noble-server-cloudimg-amd64.img // /images/noble-server-cloudimg-amd64.img
// /images/alpine-virt-cloudimg-amd64.qcow2 (Alpine prepare not implemented yet) // /images/alpine-virt-cloudimg-amd64.qcow2 (Alpine prepare not implemented yet)
// /images/hypervisor-fw (firmware binary used via --kernel) // /images/hypervisor-fw (firmware binary used via --kernel)
// - Network defaults: IPv4 NAT + dnsmasq DHCP; placeholder IPv6 on bridge + guest netplan. // - Network defaults: IPv4 NAT (dnsmasq DHCP) + IPv6 routed over Mycelium (RA/DHCPv6). No static IPv6 is written into the guest; it autoconfigures via RA.
// //
// Conventions: // Conventions:
// - Functional builder chaining: b = memory_mb(b, 4096), etc. // - Functional builder chaining: b = memory_mb(b, 4096), etc.
@@ -81,12 +81,6 @@ let prep_opts = #{
flavor: "ubuntu", flavor: "ubuntu",
// source: optional override, default uses /images/noble-server-cloudimg-amd64.img // source: optional override, default uses /images/noble-server-cloudimg-amd64.img
// target_dir: optional override, default $HOME/hero/virt/vms/<id> // target_dir: optional override, default $HOME/hero/virt/vms/<id>
net: #{
dhcp4: true,
dhcp6: false,
ipv6_addr: "400::10/64",
gw6: "400::1",
},
disable_cloud_init_net: true, disable_cloud_init_net: true,
}; };
@@ -150,6 +144,7 @@ try {
fail("cloudhv_vm_list failed: " + e.to_string()); fail("cloudhv_vm_list failed: " + e.to_string());
} }
sleep(1000000);
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
// Phase 5: Stop & delete VM A // Phase 5: Stop & delete VM A
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------