From 76876049bef6ef1c3b5985e4d5206f5035aa3cc9 Mon Sep 17 00:00:00 2001 From: Mahmoud-Emad Date: Mon, 24 Nov 2025 14:02:36 +0200 Subject: [PATCH] test: Add comprehensive heropods network and container tests - Add wait_for_process_ready to container start - Reduce sigterm and stop check timeouts - Update default container base directory - Introduce new heropods test suite with multiple tests - Add tests for initialization and custom network config - Add tests for Docker image pull and container creation - Add tests for container lifecycle (start, stop, delete) - Add tests for container command execution - Add tests for network IP allocation - Add tests for IPv4 connectivity - Add tests for container deletion and IP cleanup - Add tests for bridge network setup and NAT rules - Add tests for IP pool management - Add tests for custom bridge configuration --- lib/virt/heropods/container.v | 63 +++++- lib/virt/heropods/heropods_model.v | 2 +- lib/virt/heropods/heropods_test.v | 348 +++++++++++++++++++++++++++++ lib/virt/heropods/network_test.v | 278 +++++++++++++++++++++++ test_basic.vsh | 3 +- 5 files changed, 685 insertions(+), 9 deletions(-) create mode 100644 lib/virt/heropods/heropods_test.v create mode 100644 lib/virt/heropods/network_test.v diff --git a/lib/virt/heropods/container.v b/lib/virt/heropods/container.v index 76f542a4..071809c4 100644 --- a/lib/virt/heropods/container.v +++ b/lib/virt/heropods/container.v @@ -6,12 +6,13 @@ import incubaid.herolib.virt.crun import time import incubaid.herolib.builder import json +import os // Container lifecycle timeout constants const cleanup_retry_delay_ms = 500 // Time to wait for filesystem cleanup to complete -const sigterm_timeout_ms = 5000 // Time to wait for graceful shutdown (5 seconds) +const sigterm_timeout_ms = 1000 // Time to wait for graceful shutdown (1 second) - reduced from 5s for faster tests const sigkill_wait_ms = 500 // Time to wait after SIGKILL -const stop_check_interval_ms = 500 // Interval to check if container stopped +const stop_check_interval_ms = 200 // Interval to check if container stopped - reduced from 500ms for faster response // Container represents a running or stopped OCI container managed by crun // @@ -129,10 +130,11 @@ pub fn (mut self Container) start() ! { // start the container (crun start doesn't have --detach flag) crun_root := '${self.factory.base_dir}/runtime' - // Start the container - osal.exec(cmd: 'crun --root ${crun_root} start ${self.name}', stdout: true) or { - return error('Failed to start container: ${err}') - } + osal.exec(cmd: 'crun --root ${crun_root} start ${self.name}', stdout: true)! + + // Wait for container process to be fully ready before setting up network + // Poll for the PID and verify /proc//ns/net exists + self.wait_for_process_ready()! // Setup network for the container (thread-safe) // If this fails, stop the container to clean up @@ -419,6 +421,55 @@ pub fn (self Container) pid() !int { return state.pid } +// Wait for container process to be fully ready +// +// After `crun start` returns, the container process may not be fully initialized yet. +// This method polls for the container's PID and verifies that /proc//ns/net exists +// before returning. This ensures network setup can proceed without errors. +// +// The method uses exponential backoff polling (no sleep delays) to minimize wait time. +fn (self Container) wait_for_process_ready() ! { + crun_root := '${self.factory.base_dir}/runtime' + + // Poll for up to 100 iterations (very fast, no sleep) + // Most containers will be ready within the first few iterations + for i in 0 .. 100 { + // Try to get the container state + result := osal.exec( + cmd: 'crun --root ${crun_root} state ${self.name}' + stdout: false + ) or { + // Container state not ready yet, continue polling + continue + } + + // Parse the state to get PID + state := json.decode(CrunState, result.output) or { + // JSON not ready yet, continue polling + continue + } + + // Check if we have a valid PID + if state.pid == 0 { + continue + } + + // Verify that /proc//ns/net exists (this is what nsenter needs) + ns_net_path := '/proc/${state.pid}/ns/net' + if os.exists(ns_net_path) { + // Process is ready! + return + } + + // If we've tried many times, add a tiny yield to avoid busy-waiting + if i > 50 && i % 10 == 0 { + time.sleep(1 * time.millisecond) + } + } + + return error('Container process did not become ready in time') +} + // Setup network for this container (thread-safe) // // Delegates to HeroPods.network_setup_container() which uses network_mutex diff --git a/lib/virt/heropods/heropods_model.v b/lib/virt/heropods/heropods_model.v index 879c36eb..3f5d2fb4 100644 --- a/lib/virt/heropods/heropods_model.v +++ b/lib/virt/heropods/heropods_model.v @@ -52,7 +52,7 @@ fn obj_init(mycfg_ HeroPods) !HeroPods { // Normalize base_dir from environment variable if not set if mycfg.base_dir == '' { - mycfg.base_dir = os.getenv_opt('CONTAINERS_DIR') or { os.home_dir() + '/.containers' } + mycfg.base_dir = os.getenv_opt('CONTAINERS_DIR') or { os.home_dir() + '/.heropods/default' } } // Validate: warn if podman is requested but not available diff --git a/lib/virt/heropods/heropods_test.v b/lib/virt/heropods/heropods_test.v new file mode 100644 index 00000000..b48c6d92 --- /dev/null +++ b/lib/virt/heropods/heropods_test.v @@ -0,0 +1,348 @@ +module heropods + +import incubaid.herolib.core +import incubaid.herolib.osal.core as osal +import os + +// Simplified test suite for HeroPods container management +// +// These tests use real Docker images (Alpine Linux) for reliability +// Prerequisites: Linux, crun, podman, ip, iptables, nsenter + +// Helper function to check if we're on Linux +fn is_linux_platform() bool { + return core.is_linux() or { false } +} + +// Helper function to skip test if not on Linux +fn skip_if_not_linux() { + if !is_linux_platform() { + eprintln('SKIP: Test requires Linux (crun, ip, iptables)') + exit(0) + } +} + +// Cleanup helper for tests - stops and deletes all containers +fn cleanup_test_heropods(name string) { + mut hp := get(name: name) or { return } + + // Stop and delete all containers + for container_name, mut container in hp.containers { + container.stop() or {} + container.delete() or {} + } + + // Cleanup network - don't delete the bridge (false) - tests run in parallel + hp.network_cleanup_all(false) or {} + + // Delete from factory + delete(name: name) or {} +} + +// Test 1: HeroPods initialization and configuration +fn test_heropods_initialization() ! { + skip_if_not_linux() + + test_name := 'test_init_${os.getpid()}' + defer { + cleanup_test_heropods(test_name) + } + + mut hp := new( + name: test_name + reset: true + use_podman: true // Skip default image setup in tests + )! + + assert hp.base_dir != '' + assert hp.network_config.bridge_name == 'heropods0' + assert hp.network_config.subnet == '10.10.0.0/24' + assert hp.network_config.gateway_ip == '10.10.0.1' + assert hp.network_config.dns_servers.len > 0 + assert hp.name == test_name + + println('✓ HeroPods initialization test passed') +} + +// Test 2: Custom network configuration +fn test_custom_network_config() ! { + skip_if_not_linux() + + test_name := 'test_custom_net_${os.getpid()}' + defer { cleanup_test_heropods(test_name) } + + mut hp := new( + name: test_name + reset: true + use_podman: true // Skip default image setup in tests + bridge_name: 'testbr0' + subnet: '192.168.100.0/24' + gateway_ip: '192.168.100.1' + dns_servers: ['1.1.1.1', '1.0.0.1'] + )! + + assert hp.network_config.bridge_name == 'testbr0' + assert hp.network_config.subnet == '192.168.100.0/24' + assert hp.network_config.gateway_ip == '192.168.100.1' + assert hp.network_config.dns_servers == ['1.1.1.1', '1.0.0.1'] + + println('✓ Custom network configuration test passed') +} + +// Test 3: Pull Docker image and create container +fn test_container_creation_with_docker_image() ! { + skip_if_not_linux() + + test_name := 'test_docker_${os.getpid()}' + defer { + cleanup_test_heropods(test_name) + } + + mut hp := new( + name: test_name + reset: true + use_podman: true + )! + + container_name := 'alpine_${os.getpid()}' + + // Pull Alpine Linux image from Docker Hub (very small, ~7MB) + mut container := hp.container_new( + name: container_name + image: .custom + custom_image_name: 'alpine_test' + docker_url: 'docker.io/library/alpine:3.20' + )! + + assert container.name == container_name + assert container.factory.name == test_name + assert container_name in hp.containers + + // Verify rootfs was extracted + rootfs_path := '${hp.base_dir}/images/alpine_test/rootfs' + assert os.is_dir(rootfs_path) + // Alpine uses busybox, check for bin directory and basic structure + assert os.is_dir('${rootfs_path}/bin') + assert os.is_dir('${rootfs_path}/etc') + + println('✓ Docker image pull and container creation test passed') +} + +// Test 4: Container lifecycle with real Docker image (start, status, stop, delete) +fn test_container_lifecycle() ! { + skip_if_not_linux() + + test_name := 'test_lifecycle_${os.getpid()}' + defer { + cleanup_test_heropods(test_name) + } + + mut hp := new( + name: test_name + reset: true + use_podman: true + )! + + container_name := 'lifecycle_${os.getpid()}' + mut container := hp.container_new( + name: container_name + image: .custom + custom_image_name: 'alpine_lifecycle' + docker_url: 'docker.io/library/alpine:3.20' + )! + + // Test start + container.start()! + status := container.status()! + assert status == .running + + // Verify container has a PID + pid := container.pid()! + assert pid > 0 + + // Test stop + container.stop()! + status_after_stop := container.status()! + assert status_after_stop == .stopped + + // Test delete + container.delete()! + exists := container.container_exists_in_crun()! + assert !exists + + println('✓ Container lifecycle test passed') +} + +// Test 5: Container command execution with real Alpine image +fn test_container_exec() ! { + skip_if_not_linux() + + test_name := 'test_exec_${os.getpid()}' + defer { + cleanup_test_heropods(test_name) + } + + mut hp := new( + name: test_name + reset: true + use_podman: true + )! + + container_name := 'exec_${os.getpid()}' + mut container := hp.container_new( + name: container_name + image: .custom + custom_image_name: 'alpine_exec' + docker_url: 'docker.io/library/alpine:3.20' + )! + + container.start()! + defer { + container.stop() or {} + container.delete() or {} + } + + // Execute simple echo command + result := container.exec(cmd: 'echo "test123"')! + assert result.contains('test123') + + // Execute pwd command + result2 := container.exec(cmd: 'pwd')! + assert result2.contains('/') + + // Execute ls command (Alpine has busybox ls) + result3 := container.exec(cmd: 'ls /')! + assert result3.contains('bin') + assert result3.contains('etc') + + println('✓ Container exec test passed') +} + +// Test 6: Network IP allocation (without starting containers) +fn test_network_ip_allocation() ! { + skip_if_not_linux() + + test_name := 'test_ip_alloc_${os.getpid()}' + defer { + cleanup_test_heropods(test_name) + } + + mut hp := new( + name: test_name + reset: true + use_podman: true + )! + + // Allocate IPs for multiple containers (without starting them) + ip1 := hp.network_allocate_ip('container1')! + ip2 := hp.network_allocate_ip('container2')! + ip3 := hp.network_allocate_ip('container3')! + + // Verify IPs are different + assert ip1 != ip2 + assert ip2 != ip3 + assert ip1 != ip3 + + // Verify IPs are in correct subnet + assert ip1.starts_with('10.10.0.') + assert ip2.starts_with('10.10.0.') + assert ip3.starts_with('10.10.0.') + + // Verify IPs are tracked + assert 'container1' in hp.network_config.allocated_ips + assert 'container2' in hp.network_config.allocated_ips + assert 'container3' in hp.network_config.allocated_ips + + println('✓ Network IP allocation test passed') +} + +// Test 7: IPv4 connectivity test with real Alpine container +fn test_ipv4_connectivity() ! { + skip_if_not_linux() + + test_name := 'test_ipv4_${os.getpid()}' + defer { + cleanup_test_heropods(test_name) + } + + mut hp := new( + name: test_name + reset: true + use_podman: true + )! + + container_name := 'ipv4_${os.getpid()}' + mut container := hp.container_new( + name: container_name + image: .custom + custom_image_name: 'alpine_ipv4' + docker_url: 'docker.io/library/alpine:3.20' + )! + + container.start()! + defer { + container.stop() or {} + container.delete() or {} + } + + // Check container has an IP address + container_ip := hp.network_config.allocated_ips[container_name] or { + return error('Container should have allocated IP') + } + assert container_ip.starts_with('10.10.0.') + + // Test IPv4 connectivity by checking the container's IP configuration + result := container.exec(cmd: 'ip addr show eth0')! + assert result.contains(container_ip) + assert result.contains('eth0') + + // Test that default route exists + route_result := container.exec(cmd: 'ip route')! + assert route_result.contains('default') + assert route_result.contains('10.10.0.1') + + println('✓ IPv4 connectivity test passed') +} + +// Test 8: Container deletion and IP cleanup +fn test_container_deletion() ! { + skip_if_not_linux() + + test_name := 'test_delete_${os.getpid()}' + defer { + cleanup_test_heropods(test_name) + } + + mut hp := new( + name: test_name + reset: true + use_podman: true + )! + + container_name := 'delete_${os.getpid()}' + mut container := hp.container_new( + name: container_name + image: .custom + custom_image_name: 'alpine_delete' + docker_url: 'docker.io/library/alpine:3.20' + )! + + // Start container (allocates IP) + container.start()! + + // Verify IP is allocated + assert container_name in hp.network_config.allocated_ips + + // Stop and delete container + container.stop()! + container.delete()! + + // Verify container is deleted from crun + exists := container.container_exists_in_crun()! + assert !exists + + // Verify IP is freed + assert container_name !in hp.network_config.allocated_ips + + println('✓ Container deletion and IP cleanup test passed') +} diff --git a/lib/virt/heropods/network_test.v b/lib/virt/heropods/network_test.v new file mode 100644 index 00000000..4ba55138 --- /dev/null +++ b/lib/virt/heropods/network_test.v @@ -0,0 +1,278 @@ +module heropods + +import incubaid.herolib.core +import incubaid.herolib.osal.core as osal +import os + +// Network-specific tests for HeroPods +// +// These tests verify bridge setup, IP allocation, NAT rules, and network cleanup + +// Helper function to check if we're on Linux +fn is_linux_platform() bool { + return core.is_linux() or { false } +} + +// Helper function to skip test if not on Linux +fn skip_if_not_linux() { + if !is_linux_platform() { + eprintln('SKIP: Test requires Linux (crun, ip, iptables)') + exit(0) + } +} + +// Setup minimal test rootfs for testing +fn setup_test_rootfs() ! { + rootfs_path := os.home_dir() + '/.containers/images/alpine/rootfs' + + // Skip if already exists and has valid binaries + if os.is_dir(rootfs_path) && os.is_file('${rootfs_path}/bin/sh') { + // Check if sh is a real binary (> 1KB) + sh_info := os.stat('${rootfs_path}/bin/sh') or { return } + if sh_info.size > 1024 { + return + } + } + + // Remove old rootfs if it exists + if os.is_dir(rootfs_path) { + os.rmdir_all(rootfs_path) or {} + } + + // Create minimal rootfs structure + os.mkdir_all(rootfs_path)! + os.mkdir_all('${rootfs_path}/bin')! + os.mkdir_all('${rootfs_path}/etc')! + os.mkdir_all('${rootfs_path}/dev')! + os.mkdir_all('${rootfs_path}/proc')! + os.mkdir_all('${rootfs_path}/sys')! + os.mkdir_all('${rootfs_path}/tmp')! + os.mkdir_all('${rootfs_path}/usr/bin')! + os.mkdir_all('${rootfs_path}/usr/local/bin')! + os.mkdir_all('${rootfs_path}/lib/x86_64-linux-gnu')! + os.mkdir_all('${rootfs_path}/lib64')! + + // Copy essential binaries from host + // Use dash (smaller than bash) and sleep + if os.exists('/bin/dash') { + os.execute('cp -L /bin/dash ${rootfs_path}/bin/sh') + os.chmod('${rootfs_path}/bin/sh', 0o755)! + } else if os.exists('/bin/sh') { + os.execute('cp -L /bin/sh ${rootfs_path}/bin/sh') + os.chmod('${rootfs_path}/bin/sh', 0o755)! + } + + // Copy common utilities + for cmd in ['sleep', 'echo', 'cat', 'ls', 'pwd', 'true', 'false'] { + if os.exists('/bin/${cmd}') { + os.execute('cp -L /bin/${cmd} ${rootfs_path}/bin/${cmd}') + os.chmod('${rootfs_path}/bin/${cmd}', 0o755) or {} + } else if os.exists('/usr/bin/${cmd}') { + os.execute('cp -L /usr/bin/${cmd} ${rootfs_path}/bin/${cmd}') + os.chmod('${rootfs_path}/bin/${cmd}', 0o755) or {} + } + } + + // Copy required libraries for dash/sh + // Copy from /lib/x86_64-linux-gnu to the same path in rootfs + if os.is_dir('/lib/x86_64-linux-gnu') { + os.execute('cp -a /lib/x86_64-linux-gnu/libc.so.6 ${rootfs_path}/lib/x86_64-linux-gnu/') + os.execute('cp -a /lib/x86_64-linux-gnu/libc-*.so ${rootfs_path}/lib/x86_64-linux-gnu/ 2>/dev/null || true') + // Copy dynamic linker (actual file, not symlink) + os.execute('cp -L /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 ${rootfs_path}/lib/x86_64-linux-gnu/') + } + + // Create symlink in /lib64 pointing to the actual file + if os.is_dir('${rootfs_path}/lib64') { + os.execute('ln -sf ../lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 ${rootfs_path}/lib64/ld-linux-x86-64.so.2') + } + + // Create /etc/resolv.conf + os.write_file('${rootfs_path}/etc/resolv.conf', 'nameserver 8.8.8.8\n')! +} + +// Cleanup helper for tests +fn cleanup_test_heropods(name string) { + mut hp := get(name: name) or { return } + for container_name, mut container in hp.containers { + container.stop() or {} + container.delete() or {} + } + // Don't delete the bridge (false) - tests run in parallel and share the same bridge + // Only clean up containers and IPs + hp.network_cleanup_all(false) or {} + delete(name: name) or {} +} + +// Test 1: Bridge network setup +fn test_network_bridge_setup() ! { + skip_if_not_linux() + + test_name := 'test_bridge_${os.getpid()}' + + mut hp := new( + name: test_name + reset: true + use_podman: true // Skip default image setup in tests + )! + + bridge_name := hp.network_config.bridge_name + + // Verify bridge exists + job := osal.exec(cmd: 'ip link show ${bridge_name}')! + assert job.output.contains(bridge_name) + + // Verify bridge is UP + assert job.output.contains('UP') || job.output.contains('state UP') + + // Verify IP is assigned to bridge + job2 := osal.exec(cmd: 'ip addr show ${bridge_name}')! + assert job2.output.contains(hp.network_config.gateway_ip) + + // Cleanup after test + cleanup_test_heropods(test_name) + + println('✓ Bridge network setup test passed') +} + +// Test 2: NAT rules verification +fn test_network_nat_rules() ! { + skip_if_not_linux() + + test_name := 'test_nat_${os.getpid()}' + defer { cleanup_test_heropods(test_name) } + + mut hp := new( + name: test_name + reset: true + use_podman: true // Skip default image setup in tests + )! + + // Verify NAT rules exist for the subnet + job := osal.exec(cmd: 'iptables -t nat -L POSTROUTING -n')! + assert job.output.contains(hp.network_config.subnet) || job.output.contains('MASQUERADE') + + println('✓ NAT rules test passed') +} + +// Test 3: IP allocation sequential +fn test_ip_allocation_sequential() ! { + skip_if_not_linux() + + test_name := 'test_ip_seq_${os.getpid()}' + defer { cleanup_test_heropods(test_name) } + + mut hp := new( + name: test_name + reset: true + use_podman: true // Skip default image setup in tests + )! + + // Allocate multiple IPs + mut allocated_ips := []string{} + for i in 0 .. 10 { + ip := hp.network_allocate_ip('container_${i}')! + allocated_ips << ip + } + + // Verify all IPs are unique + for i, ip1 in allocated_ips { + for j, ip2 in allocated_ips { + if i != j { + assert ip1 != ip2, 'IPs should be unique: ${ip1} == ${ip2}' + } + } + } + + // Verify all IPs are in correct subnet + for ip in allocated_ips { + assert ip.starts_with('10.10.0.') + } + + println('✓ IP allocation sequential test passed') +} + +// Test 4: IP pool management with container lifecycle +fn test_ip_pool_management() ! { + skip_if_not_linux() + setup_test_rootfs()! + + test_name := 'test_ip_pool_${os.getpid()}' + defer { cleanup_test_heropods(test_name) } + + mut hp := new( + name: test_name + reset: true + use_podman: true // Skip default image setup in tests + )! + + // Create and start 3 containers + mut container1 := hp.container_new(name: 'pool_test1_${os.getpid()}', image: .alpine_3_20)! + mut container2 := hp.container_new(name: 'pool_test2_${os.getpid()}', image: .alpine_3_20)! + mut container3 := hp.container_new(name: 'pool_test3_${os.getpid()}', image: .alpine_3_20)! + + container1.start()! + container2.start()! + container3.start()! + + // Get allocated IPs + ip1 := hp.network_config.allocated_ips[container1.name] + ip2 := hp.network_config.allocated_ips[container2.name] + ip3 := hp.network_config.allocated_ips[container3.name] + + // Delete middle container (frees IP2) + container2.stop()! + container2.delete()! + + // Verify IP2 is freed + assert container2.name !in hp.network_config.allocated_ips + + // Create new container - should reuse freed IP2 + mut container4 := hp.container_new(name: 'pool_test4_${os.getpid()}', image: .alpine_3_20)! + container4.start()! + + ip4 := hp.network_config.allocated_ips[container4.name] + assert ip4 == ip2, 'Should reuse freed IP: ${ip2} vs ${ip4}' + + // Cleanup + container1.stop()! + container1.delete()! + container3.stop()! + container3.delete()! + container4.stop()! + container4.delete()! + + println('✓ IP pool management test passed') +} + +// Test 5: Custom bridge configuration +fn test_custom_bridge_config() ! { + skip_if_not_linux() + + test_name := 'test_custom_br_${os.getpid()}' + custom_bridge := 'custombr_${os.getpid()}' + defer { + cleanup_test_heropods(test_name) + // Cleanup custom bridge + osal.exec(cmd: 'ip link delete ${custom_bridge}') or {} + } + + mut hp := new( + name: test_name + reset: true + use_podman: true // Skip default image setup in tests + bridge_name: custom_bridge + subnet: '172.20.0.0/24' + gateway_ip: '172.20.0.1' + )! + + // Verify custom bridge exists + job := osal.exec(cmd: 'ip link show ${custom_bridge}')! + assert job.output.contains(custom_bridge) + + // Verify custom IP + job2 := osal.exec(cmd: 'ip addr show ${custom_bridge}')! + assert job2.output.contains('172.20.0.1') + + println('✓ Custom bridge configuration test passed') +} diff --git a/test_basic.vsh b/test_basic.vsh index b5c5dd64..52e6270d 100755 --- a/test_basic.vsh +++ b/test_basic.vsh @@ -170,8 +170,7 @@ lib/clients lib/core lib/develop lib/hero/heromodels -// lib/vfs The vfs folder is not exists on the development branch, so we need to uncomment it after merging this PR https://github.com/incubaid/herolib/pull/68 -// lib/crypt +lib/virt/heropods ' // the following tests have no prio and can be ignored