diff --git a/packages/system/virt/src/cloudhv/mod.rs b/packages/system/virt/src/cloudhv/mod.rs index 741fc24..30ae728 100644 --- a/packages/system/virt/src/cloudhv/mod.rs +++ b/packages/system/virt/src/cloudhv/mod.rs @@ -408,27 +408,35 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { let dhcp_end = std::env::var("HERO_VIRT_DHCP_END").unwrap_or_else(|_| "172.30.0.250".into()); - // Optional IPv6 over Mycelium provisioning (bridge P::2/64 + RA) guarded by env and detection. + // IPv6 over Mycelium: enabled by default. + // If explicitly disabled via HERO_VIRT_IPV6_ENABLE=false|0, we skip. + // If enabled but Mycelium is not detected, return an error. let ipv6_env = std::env::var("HERO_VIRT_IPV6_ENABLE").unwrap_or_else(|_| "".into()); - let mut ipv6_enabled = ipv6_env.eq_ignore_ascii_case("1") || ipv6_env.eq_ignore_ascii_case("true"); + let ipv6_requested = match ipv6_env.to_lowercase().as_str() { + "" | "1" | "true" | "yes" => true, + "0" | "false" | "no" => false, + _ => true, + }; let mycelium_if_cfg = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| "mycelium".into()); let mut ipv6_bridge_cidr: Option = None; let mut mycelium_if_opt: Option = None; - let enable_auto = ipv6_env.is_empty(); // auto-enable if mycelium is detected and not explicitly disabled - if ipv6_enabled || enable_auto { + if ipv6_requested { if let Ok(cidr) = std::env::var("HERO_VIRT_IPV6_BRIDGE_CIDR") { - // Explicit override for bridge IPv6 (e.g., "400:...::2/64") + // Explicit override for bridge IPv6 (e.g., "400:...::2/64") but still require mycelium iface presence. + // Validate mycelium interface and that it has IPv6 configured. + let _ = get_mycelium_ipv6_addr(&mycelium_if_cfg)?; // returns DependencyMissing on failure ipv6_bridge_cidr = Some(cidr); mycelium_if_opt = Some(mycelium_if_cfg.clone()); - ipv6_enabled = true; - } else if let Ok((ifname, myc_addr)) = get_mycelium_ipv6_addr(&mycelium_if_cfg) { - // Derive P::2/64 from the mycelium node address - if let Ok((_pfx, router_cidr)) = derive_ipv6_prefix_from_mycelium(&myc_addr) { - ipv6_bridge_cidr = Some(router_cidr); - mycelium_if_opt = Some(ifname); - ipv6_enabled = true; - } + } else { + // Auto-derive from mycelium node address; error out if not detected. + println!("auto-deriving mycelium address..."); + let (ifname, myc_addr) = get_mycelium_ipv6_addr(&mycelium_if_cfg)?; + println!("on if {ifname}, got myc addr: {myc_addr}"); + let (_pfx, router_cidr) = derive_ipv6_prefix_from_mycelium(&myc_addr)?; + println!("derived pfx: {_pfx} and router cidr: {router_cidr}"); + ipv6_bridge_cidr = Some(router_cidr); + mycelium_if_opt = Some(ifname); } } @@ -445,8 +453,10 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { // Ensure a TAP device for this VM and attach to the bridge let tap_name = ensure_tap_for_vm(&bridge_name, id)?; + println!("TAP device for vm called: {tap_name}"); // Stable locally-administered MAC derived from VM id let mac = stable_mac_from_id(id); + println!("MAC for vm: {mac}"); parts.push("--net".into()); parts.push(format!("tap={},mac={}", tap_name, mac)); @@ -465,6 +475,7 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { log_file, vm_pid_path(id).to_string_lossy() ); + println!("executing command:\n{heredoc}"); // Execute command; this will background cloud-hypervisor and return let result = sal_process::run(&heredoc).execute(); match result { @@ -489,6 +500,7 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { Ok(s) => s.trim().parse::().ok(), Err(_) => None, }; + println!("reading PID back: {} - (if 0 == not found)", pid.unwrap_or(0)); // Quick health check: ensure process did not exit immediately due to CLI errors (e.g., duplicate flags) if let Some(pid_num) = pid { @@ -496,6 +508,7 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { if !proc_exists(pid_num) { // Tail log to surface the error cause let tail_cmd = format!("tail -n 200 {}", shell_escape(&log_file)); + println!("executing tail_cmd command:\n{tail_cmd}"); let tail = sal_process::run(&tail_cmd).die(false).silent(true).execute(); let mut log_snip = String::new(); if let Ok(res) = tail { @@ -524,6 +537,76 @@ pub fn vm_start(id: &str) -> Result<(), CloudHvError> { let value = serde_json::to_value(&rec).map_err(|e| CloudHvError::JsonError(e.to_string()))?; write_json(&vm_json_path(id), &value)?; + println!("wrote JSON for VM"); + + // Best-effort: discover and print guest IPv4/IPv6 addresses (default-net path) + // Give DHCP/ND a moment + println!("waiting 5 secs for DHCP/ND"); + thread::sleep(Duration::from_millis(5000)); + let bridge_name = std::env::var("HERO_VIRT_BRIDGE_NAME").unwrap_or_else(|_| "br-hero".into()); + let mac_lower = stable_mac_from_id(id).to_lowercase(); + + // IPv4 from dnsmasq leases (pinned per-bridge leasefile) + // Path set in ensure_host_net_prereq_dnsmasq_nftables: /var/lib/misc/dnsmasq-hero-$BR.leases + let lease_path = std::env::var("HERO_VIRT_DHCP_LEASE_FILE") + .unwrap_or_else(|_| format!("/var/lib/misc/dnsmasq-hero-{}.leases", bridge_name)); + // Parse dnsmasq leases directly to avoid shell quoting/pipelines + let ipv4 = (|| { + let deadline = std::time::Instant::now() + Duration::from_secs(12); + loop { + if let Ok(content) = fs::read_to_string(&lease_path) { + let mut last_ip: Option = None; + for line in content.lines() { + let cols: Vec<&str> = line.split_whitespace().collect(); + if cols.len() >= 3 && cols[1].eq_ignore_ascii_case(&mac_lower) { + last_ip = Some(cols[2].to_string()); + } + } + if last_ip.is_some() { + return last_ip; + } + } + if std::time::Instant::now() >= deadline { + return None; + } + thread::sleep(Duration::from_millis(800)); + } + })(); + println!( + "Got IPv4 from dnsmasq lease ({}): {}", + lease_path, + ipv4.clone().unwrap_or("not found".to_string()) + ); + + // IPv6 from neighbor table on the bridge (exclude link-local), parsed in Rust + let ipv6 = (|| { + let cmd = format!("ip -6 neigh show dev {}", bridge_name); + if let Ok(res) = sal_process::run(&cmd).silent(true).die(false).execute() { + if res.success { + let mac_pat = format!("lladdr {}", mac_lower); + for line in res.stdout.lines() { + let lt = line.trim(); + if lt.to_lowercase().contains(&mac_pat) { + if let Some(addr) = lt.split_whitespace().next() { + if !addr.starts_with("fe80") && !addr.is_empty() { + return Some(addr.to_string()); + } + } + } + } + } + } + None + })(); + + println!("Got IPv6 from neighbor table on bridge: {}", ipv6.clone().unwrap_or("not found".to_string())); + + println!( + "[cloudhv] VM '{}' guest addresses: IPv4={}, IPv6={}", + id, + ipv4.as_deref().unwrap_or(""), + ipv6.as_deref().unwrap_or("") + ); Ok(()) } @@ -711,50 +794,44 @@ fn stable_mac_from_id(id: &str) -> String { format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", b0, b1, b2, b3, b4, b5) } -/// Discover the mycelium IPv6 address and validate the interface. -/// -/// Returns (interface_name, address_string). +/// Discover the mycelium IPv6 address by inspecting the interface itself (no CLI dependency). +/// Returns (interface_name, first global IPv6 address found on the interface). fn get_mycelium_ipv6_addr(iface_hint: &str) -> Result<(String, String), CloudHvError> { - // Parse `mycelium inspect` for "Address: ..." - let insp = sal_process::run("mycelium inspect").silent(true).die(false).execute(); - let mut addr: Option = None; - if let Ok(res) = insp { - if res.success { - for l in res.stdout.lines() { - let lt = l.trim(); - if let Some(rest) = lt.strip_prefix("Address:") { - let val = rest.trim().trim_matches('"').to_string(); - if !val.is_empty() { - addr = Some(val); - break; - } - } - } - } - } let iface = std::env::var("HERO_VIRT_MYCELIUM_IF").unwrap_or_else(|_| iface_hint.to_string()); - // Validate interface exists and has IPv6 configured (best-effort) + // Query IPv6 addresses on the interface let cmd = format!("ip -6 addr show dev {}", shell_escape(&iface)); - match sal_process::run(&cmd).silent(true).die(false).execute() { - Ok(r) if r.success => { - // proceed - } + let res = sal_process::run(&cmd).silent(true).die(false).execute(); + let out = match res { + Ok(r) if r.success => r.stdout, _ => { return Err(CloudHvError::DependencyMissing(format!( "mycelium interface '{}' not found or no IPv6 configured", iface - ))); + ))) + } + }; + + // Extract the first global IPv6 address present on the interface. + for line in out.lines() { + let lt = line.trim(); + // Example line: "inet6 578:9fcf:.../7 scope global" + if lt.starts_with("inet6 ") && lt.contains("scope global") { + let parts: Vec<&str> = lt.split_whitespace().collect(); + if let Some(addr_cidr) = parts.get(1) { + let addr_only = addr_cidr.split('/').next().unwrap_or("").trim(); + println!("got addr from host: {addr_only}"); + if !addr_only.is_empty() && addr_only.parse::().is_ok() { + return Ok((iface, addr_only.to_string())); + } + } } } - if let Some(a) = addr { - Ok((iface, a)) - } else { - Err(CloudHvError::DependencyMissing( - "failed to read mycelium IPv6 address via `mycelium inspect`".into(), - )) - } + Err(CloudHvError::DependencyMissing(format!( + "no global IPv6 found on interface '{}'", + iface + ))) } /// Derive a /64 prefix P from the mycelium IPv6 and return (P/64, P::2/64). @@ -803,10 +880,15 @@ SUBNET={subnet} DHCP_START={dstart} DHCP_END={dend} IPV6_CIDR={v6cidr} +LEASE_FILE=/var/lib/misc/dnsmasq-hero-$BR.leases # Determine default WAN interface WAN_IF=$(ip -o route show default | awk '{{print $5}}' | head -n1) - +if [ -z \"$WAN_IF\" ]; then + echo \"No default WAN interface detected (required for IPv4 NAT)\" >&2 + exit 2 +fi + # Bridge creation (idempotent) ip link show \"$BR\" >/dev/null 2>&1 || ip link add name \"$BR\" type bridge ip addr replace \"$BR_ADDR\" dev \"$BR\" @@ -829,24 +911,38 @@ nft list chain ip hero postrouting | grep -q \"ip saddr $SUBNET oifname \\\"$WAN # dnsmasq DHCPv4 + RA/DHCPv6 config (idempotent) mkdir -p /etc/dnsmasq.d +mkdir -p /var/lib/misc CFG=/etc/dnsmasq.d/hero-$BR.conf TMP=/etc/dnsmasq.d/.hero-$BR.conf.new -# Always include IPv4 section -cat >\"$TMP\" <> \"$CONF\" + RELOAD=1 +fi + +# Ensure lease file exists and is writable by dnsmasq user +touch \"$LEASE_FILE\" || true +chown dnsmasq:dnsmasq \"$LEASE_FILE\" 2>/dev/null || true + +# Always include IPv4 section +printf '%s\n' \ + \"interface=$BR\" \ + \"bind-interfaces\" \ + \"dhcp-authoritative\" \ + \"dhcp-range=$DHCP_START,$DHCP_END,12h\" \ + \"dhcp-option=option:dns-server,1.1.1.1,8.8.8.8\" \ + \"dhcp-leasefile=$LEASE_FILE\" >\"$TMP\" + # Optionally append IPv6 RA/DHCPv6 if [ -n \"$IPV6_CIDR\" ]; then - cat >>\"$TMP\" <<'EOFV6' -enable-ra -dhcp-range=::,constructor:BR_PLACEHOLDER,ra-names,64,12h -dhcp-option=option6:dns-server,[2001:4860:4860::8888],[2606:4700:4700::1111] -EOFV6 + printf '%s\n' \ + \"enable-ra\" \ + \"dhcp-range=::,constructor:BR_PLACEHOLDER,ra-names,64,12h\" \ + \"dhcp-option=option6:dns-server,[2001:4860:4860::8888],[2606:4700:4700::1111]\" >>\"$TMP\" sed -i \"s/BR_PLACEHOLDER/$BR/g\" \"$TMP\" fi @@ -861,6 +957,11 @@ else rm -f \"$TMP\" systemctl enable --now dnsmasq || true fi + +# Reload if main conf was updated to include conf-dir +if [ \"$RELOAD\" = \"1\" ]; then + systemctl reload dnsmasq || systemctl restart dnsmasq || true +fi ", br = shell_escape(bridge_name), br_addr = shell_escape(bridge_addr_cidr), @@ -872,6 +973,7 @@ fi // Use a unique heredoc delimiter to avoid clashing with inner < Ok(()), diff --git a/packages/system/virt/src/image_prep/mod.rs b/packages/system/virt/src/image_prep/mod.rs index 8b403b3..7050661 100644 --- a/packages/system/virt/src/image_prep/mod.rs +++ b/packages/system/virt/src/image_prep/mod.rs @@ -4,6 +4,9 @@ use std::path::Path; use sal_os; use sal_process; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::net::Ipv6Addr; #[derive(Debug)] pub enum ImagePrepError { @@ -80,6 +83,19 @@ fn default_disable_cloud_init_net() -> bool { true } +fn stable_mac_from_id(id: &str) -> String { + let mut h = DefaultHasher::new(); + id.hash(&mut h); + let v = h.finish(); + let b0 = (((v >> 40) & 0xff) as u8 & 0xfe) | 0x02; // locally administered, unicast + let b1 = ((v >> 32) & 0xff) as u8; + let b2 = ((v >> 24) & 0xff) as u8; + let b3 = ((v >> 16) & 0xff) as u8; + let b4 = ((v >> 8) & 0xff) as u8; + let b5 = (v & 0xff) as u8; + format!("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", b0, b1, b2, b3, b4, b5) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ImagePrepResult { pub raw_disk: String, @@ -156,7 +172,67 @@ pub fn image_prepare(opts: &ImagePrepOptions) -> Result = None; + if static_v6 { + let cmd = format!("ip -6 addr show dev {}", shell_escape(&myc_if)); + if let Ok(r) = sal_process::run(&cmd).silent(true).die(false).execute() { + if r.success { + for l in r.stdout.lines() { + let lt = l.trim(); + if lt.starts_with("inet6 ") && lt.contains("scope global") { + if let Some(addr_cidr) = lt.split_whitespace().nth(1) { + let addr_only = addr_cidr.split('/').next().unwrap_or("").trim(); + if let Ok(ip) = addr_only.parse::() { + let seg0 = ip.segments()[0]; + if (seg0 & 0xFE00) == 0x0400 { + host_v6 = Some(ip); + break; + } + } + } + } + } + } + } + } + + // Derive per-host /64 from mycelium and deterministic per-VM guest address + let mut np_v6_block = String::new(); + let mut dhcp6_effective = opts.net.dhcp6; + if static_v6 { + if let Some(h) = host_v6 { + let seg = h.segments(); + // Router = P::2; Guest address = P:: + let mut hasher = DefaultHasher::new(); + opts.id.hash(&mut hasher); + let mut suffix = (hasher.finish() as u16) & 0xfffe; + if suffix == 0 || suffix == 2 { + suffix = 0x100; + } + let guest_ip = Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, suffix).to_string(); + let gw_ip = Ipv6Addr::new(seg[0], seg[1], seg[2], seg[3], 0, 0, 0, 2).to_string(); + + // Inject a YAML block for static v6 + np_v6_block = format!( + " addresses:\n - {}/64\n routes:\n - to: \"::/0\"\n via: {}\n", + guest_ip, gw_ip + ); + // Disable dhcp6 when we provide a static address + dhcp6_effective = false; + } + } + // Keep script small and robust; avoid brace-heavy awk to simplify escaping. + // Compute stable MAC (must match what vm_start() uses) and use it to match NIC in netplan. + let vm_mac = stable_mac_from_id(&opts.id); let script = format!( "#!/bin/bash -e set -euo pipefail @@ -328,11 +404,69 @@ network: version: 2 ethernets: eth0: + match: + macaddress: {vm_mac} + set-name: eth0 dhcp4: {dhcp4} dhcp6: {dhcp6} - nameservers: +{np_v6_block} nameservers: addresses: [8.8.8.8, 1.1.1.1, 2001:4860:4860::8888] EOF +# Enable SSH password authentication and set a default password for 'ubuntu' +mkdir -p \"$MNT_ROOT/etc/cloud/cloud.cfg.d\" +printf '%s\n' 'ssh_pwauth: true' > \"$MNT_ROOT/etc/cloud/cloud.cfg.d/99-ssh-password-auth.cfg\" + +mkdir -p \"$MNT_ROOT/etc/ssh/sshd_config.d\" +cat > \"$MNT_ROOT/etc/ssh/sshd_config.d/99-hero-password-auth.conf\" << EOF +PasswordAuthentication yes +KbdInteractiveAuthentication yes +UsePAM yes +EOF + +# Set password for default user 'ubuntu' +if chroot \"$MNT_ROOT\" getent passwd ubuntu >/dev/null 2>&1; then + chroot \"$MNT_ROOT\" bash -c \"echo 'ubuntu:ubuntu' | chpasswd\" || true +fi +# Ensure openssh-server is present (some cloud images may omit it) +# Ensure SSH service enabled and keys generated on boot +chroot \"$MNT_ROOT\" systemctl unmask ssh 2>/dev/null || true +chroot \"$MNT_ROOT\" systemctl enable ssh 2>/dev/null || true +chroot \"$MNT_ROOT\" systemctl enable ssh-keygen.service 2>/dev/null || true + +# Ensure sshd listens on both IPv4 and IPv6 explicitly +cat > \"$MNT_ROOT/etc/ssh/sshd_config.d/99-hero-address-family.conf\" << EOF +AddressFamily any +ListenAddress :: +ListenAddress 0.0.0.0 +EOF + +# If UFW is present, allow SSH and disable firewall (for tests) +if chroot \"$MNT_ROOT\" command -v ufw >/dev/null 2>&1; then + chroot \"$MNT_ROOT\" ufw allow OpenSSH || true + chroot \"$MNT_ROOT\" ufw disable || true +fi +if ! chroot \"$MNT_ROOT\" test -x /usr/sbin/sshd; then + cp -f /etc/resolv.conf \"$MNT_ROOT/etc/resolv.conf\" 2>/dev/null || true + chroot \"$MNT_ROOT\" bash -c \"apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server\" || true +fi +# Ensure default user 'ubuntu' exists (fallback for minimal images) +if ! chroot \"$MNT_ROOT\" id -u ubuntu >/dev/null 2>&1; then + chroot \"$MNT_ROOT\" useradd -m -s /bin/bash ubuntu || true + echo \"ubuntu ALL=(ALL) NOPASSWD:ALL\" > \"$MNT_ROOT/etc/sudoers.d/90-ubuntu\" || true + chmod 0440 \"$MNT_ROOT/etc/sudoers.d/90-ubuntu\" || true +fi + +# Re-assert password (covers both existing and newly created users) +if chroot \"$MNT_ROOT\" getent passwd ubuntu >/dev/null 2>&1; then + chroot \"$MNT_ROOT\" bash -c \"echo 'ubuntu:ubuntu' | chpasswd\" || true +fi + +# Pre-generate host SSH keys so sshd can start immediately +chroot \"$MNT_ROOT\" ssh-keygen -A 2>/dev/null || true +mkdir -p \"$MNT_ROOT/var/run/sshd\" + +# Also enable socket activation as a fallback +chroot \"$MNT_ROOT\" systemctl enable ssh.socket 2>/dev/null || true # Disable cloud-init networking (optional but default) if [ \"{disable_ci_net}\" = \"true\" ]; then @@ -361,11 +495,16 @@ exit 0 mnt_root = shell_escape(&mnt_root), mnt_boot = shell_escape(&mnt_boot), raw = shell_escape(&raw_path), + vm_mac = vm_mac, dhcp4 = if opts.net.dhcp4 { "true" } else { "false" }, - dhcp6 = if opts.net.dhcp6 { "true" } else { "false" }, + dhcp6 = if dhcp6_effective { "true" } else { "false" }, + np_v6_block = np_v6_block, disable_ci_net = if disable_ci_net { "true" } else { "false" }, ); + // image prep script printout for debugging: + println!("{script}"); + let res = run_script(&script)?; // Prefer a RESULT:-prefixed line (robust against extra stdout noise) let mut marker: Option = None;