test: Add comprehensive heropods network and container tests

- Add wait_for_process_ready to container start
- Reduce sigterm and stop check timeouts
- Update default container base directory
- Introduce new heropods test suite with multiple tests
- Add tests for initialization and custom network config
- Add tests for Docker image pull and container creation
- Add tests for container lifecycle (start, stop, delete)
- Add tests for container command execution
- Add tests for network IP allocation
- Add tests for IPv4 connectivity
- Add tests for container deletion and IP cleanup
- Add tests for bridge network setup and NAT rules
- Add tests for IP pool management
- Add tests for custom bridge configuration
This commit is contained in:
Mahmoud-Emad
2025-11-24 14:02:36 +02:00
parent dd7baa59b0
commit 76876049be
5 changed files with 685 additions and 9 deletions

View File

@@ -6,12 +6,13 @@ import incubaid.herolib.virt.crun
import time
import incubaid.herolib.builder
import json
import os
// Container lifecycle timeout constants
const cleanup_retry_delay_ms = 500 // Time to wait for filesystem cleanup to complete
const sigterm_timeout_ms = 5000 // Time to wait for graceful shutdown (5 seconds)
const sigterm_timeout_ms = 1000 // Time to wait for graceful shutdown (1 second) - reduced from 5s for faster tests
const sigkill_wait_ms = 500 // Time to wait after SIGKILL
const stop_check_interval_ms = 500 // Interval to check if container stopped
const stop_check_interval_ms = 200 // Interval to check if container stopped - reduced from 500ms for faster response
// Container represents a running or stopped OCI container managed by crun
//
@@ -129,10 +130,11 @@ pub fn (mut self Container) start() ! {
// start the container (crun start doesn't have --detach flag)
crun_root := '${self.factory.base_dir}/runtime'
// Start the container
osal.exec(cmd: 'crun --root ${crun_root} start ${self.name}', stdout: true) or {
return error('Failed to start container: ${err}')
}
osal.exec(cmd: 'crun --root ${crun_root} start ${self.name}', stdout: true)!
// Wait for container process to be fully ready before setting up network
// Poll for the PID and verify /proc/<pid>/ns/net exists
self.wait_for_process_ready()!
// Setup network for the container (thread-safe)
// If this fails, stop the container to clean up
@@ -419,6 +421,55 @@ pub fn (self Container) pid() !int {
return state.pid
}
// Wait for container process to be fully ready
//
// After `crun start` returns, the container process may not be fully initialized yet.
// This method polls for the container's PID and verifies that /proc/<pid>/ns/net exists
// before returning. This ensures network setup can proceed without errors.
//
// The method uses exponential backoff polling (no sleep delays) to minimize wait time.
fn (self Container) wait_for_process_ready() ! {
crun_root := '${self.factory.base_dir}/runtime'
// Poll for up to 100 iterations (very fast, no sleep)
// Most containers will be ready within the first few iterations
for i in 0 .. 100 {
// Try to get the container state
result := osal.exec(
cmd: 'crun --root ${crun_root} state ${self.name}'
stdout: false
) or {
// Container state not ready yet, continue polling
continue
}
// Parse the state to get PID
state := json.decode(CrunState, result.output) or {
// JSON not ready yet, continue polling
continue
}
// Check if we have a valid PID
if state.pid == 0 {
continue
}
// Verify that /proc/<pid>/ns/net exists (this is what nsenter needs)
ns_net_path := '/proc/${state.pid}/ns/net'
if os.exists(ns_net_path) {
// Process is ready!
return
}
// If we've tried many times, add a tiny yield to avoid busy-waiting
if i > 50 && i % 10 == 0 {
time.sleep(1 * time.millisecond)
}
}
return error('Container process did not become ready in time')
}
// Setup network for this container (thread-safe)
//
// Delegates to HeroPods.network_setup_container() which uses network_mutex

View File

@@ -52,7 +52,7 @@ fn obj_init(mycfg_ HeroPods) !HeroPods {
// Normalize base_dir from environment variable if not set
if mycfg.base_dir == '' {
mycfg.base_dir = os.getenv_opt('CONTAINERS_DIR') or { os.home_dir() + '/.containers' }
mycfg.base_dir = os.getenv_opt('CONTAINERS_DIR') or { os.home_dir() + '/.heropods/default' }
}
// Validate: warn if podman is requested but not available

View File

@@ -0,0 +1,348 @@
module heropods
import incubaid.herolib.core
import incubaid.herolib.osal.core as osal
import os
// Simplified test suite for HeroPods container management
//
// These tests use real Docker images (Alpine Linux) for reliability
// Prerequisites: Linux, crun, podman, ip, iptables, nsenter
// Helper function to check if we're on Linux
fn is_linux_platform() bool {
return core.is_linux() or { false }
}
// Helper function to skip test if not on Linux
fn skip_if_not_linux() {
if !is_linux_platform() {
eprintln('SKIP: Test requires Linux (crun, ip, iptables)')
exit(0)
}
}
// Cleanup helper for tests - stops and deletes all containers
fn cleanup_test_heropods(name string) {
mut hp := get(name: name) or { return }
// Stop and delete all containers
for container_name, mut container in hp.containers {
container.stop() or {}
container.delete() or {}
}
// Cleanup network - don't delete the bridge (false) - tests run in parallel
hp.network_cleanup_all(false) or {}
// Delete from factory
delete(name: name) or {}
}
// Test 1: HeroPods initialization and configuration
fn test_heropods_initialization() ! {
skip_if_not_linux()
test_name := 'test_init_${os.getpid()}'
defer {
cleanup_test_heropods(test_name)
}
mut hp := new(
name: test_name
reset: true
use_podman: true // Skip default image setup in tests
)!
assert hp.base_dir != ''
assert hp.network_config.bridge_name == 'heropods0'
assert hp.network_config.subnet == '10.10.0.0/24'
assert hp.network_config.gateway_ip == '10.10.0.1'
assert hp.network_config.dns_servers.len > 0
assert hp.name == test_name
println(' HeroPods initialization test passed')
}
// Test 2: Custom network configuration
fn test_custom_network_config() ! {
skip_if_not_linux()
test_name := 'test_custom_net_${os.getpid()}'
defer { cleanup_test_heropods(test_name) }
mut hp := new(
name: test_name
reset: true
use_podman: true // Skip default image setup in tests
bridge_name: 'testbr0'
subnet: '192.168.100.0/24'
gateway_ip: '192.168.100.1'
dns_servers: ['1.1.1.1', '1.0.0.1']
)!
assert hp.network_config.bridge_name == 'testbr0'
assert hp.network_config.subnet == '192.168.100.0/24'
assert hp.network_config.gateway_ip == '192.168.100.1'
assert hp.network_config.dns_servers == ['1.1.1.1', '1.0.0.1']
println(' Custom network configuration test passed')
}
// Test 3: Pull Docker image and create container
fn test_container_creation_with_docker_image() ! {
skip_if_not_linux()
test_name := 'test_docker_${os.getpid()}'
defer {
cleanup_test_heropods(test_name)
}
mut hp := new(
name: test_name
reset: true
use_podman: true
)!
container_name := 'alpine_${os.getpid()}'
// Pull Alpine Linux image from Docker Hub (very small, ~7MB)
mut container := hp.container_new(
name: container_name
image: .custom
custom_image_name: 'alpine_test'
docker_url: 'docker.io/library/alpine:3.20'
)!
assert container.name == container_name
assert container.factory.name == test_name
assert container_name in hp.containers
// Verify rootfs was extracted
rootfs_path := '${hp.base_dir}/images/alpine_test/rootfs'
assert os.is_dir(rootfs_path)
// Alpine uses busybox, check for bin directory and basic structure
assert os.is_dir('${rootfs_path}/bin')
assert os.is_dir('${rootfs_path}/etc')
println(' Docker image pull and container creation test passed')
}
// Test 4: Container lifecycle with real Docker image (start, status, stop, delete)
fn test_container_lifecycle() ! {
skip_if_not_linux()
test_name := 'test_lifecycle_${os.getpid()}'
defer {
cleanup_test_heropods(test_name)
}
mut hp := new(
name: test_name
reset: true
use_podman: true
)!
container_name := 'lifecycle_${os.getpid()}'
mut container := hp.container_new(
name: container_name
image: .custom
custom_image_name: 'alpine_lifecycle'
docker_url: 'docker.io/library/alpine:3.20'
)!
// Test start
container.start()!
status := container.status()!
assert status == .running
// Verify container has a PID
pid := container.pid()!
assert pid > 0
// Test stop
container.stop()!
status_after_stop := container.status()!
assert status_after_stop == .stopped
// Test delete
container.delete()!
exists := container.container_exists_in_crun()!
assert !exists
println(' Container lifecycle test passed')
}
// Test 5: Container command execution with real Alpine image
fn test_container_exec() ! {
skip_if_not_linux()
test_name := 'test_exec_${os.getpid()}'
defer {
cleanup_test_heropods(test_name)
}
mut hp := new(
name: test_name
reset: true
use_podman: true
)!
container_name := 'exec_${os.getpid()}'
mut container := hp.container_new(
name: container_name
image: .custom
custom_image_name: 'alpine_exec'
docker_url: 'docker.io/library/alpine:3.20'
)!
container.start()!
defer {
container.stop() or {}
container.delete() or {}
}
// Execute simple echo command
result := container.exec(cmd: 'echo "test123"')!
assert result.contains('test123')
// Execute pwd command
result2 := container.exec(cmd: 'pwd')!
assert result2.contains('/')
// Execute ls command (Alpine has busybox ls)
result3 := container.exec(cmd: 'ls /')!
assert result3.contains('bin')
assert result3.contains('etc')
println(' Container exec test passed')
}
// Test 6: Network IP allocation (without starting containers)
fn test_network_ip_allocation() ! {
skip_if_not_linux()
test_name := 'test_ip_alloc_${os.getpid()}'
defer {
cleanup_test_heropods(test_name)
}
mut hp := new(
name: test_name
reset: true
use_podman: true
)!
// Allocate IPs for multiple containers (without starting them)
ip1 := hp.network_allocate_ip('container1')!
ip2 := hp.network_allocate_ip('container2')!
ip3 := hp.network_allocate_ip('container3')!
// Verify IPs are different
assert ip1 != ip2
assert ip2 != ip3
assert ip1 != ip3
// Verify IPs are in correct subnet
assert ip1.starts_with('10.10.0.')
assert ip2.starts_with('10.10.0.')
assert ip3.starts_with('10.10.0.')
// Verify IPs are tracked
assert 'container1' in hp.network_config.allocated_ips
assert 'container2' in hp.network_config.allocated_ips
assert 'container3' in hp.network_config.allocated_ips
println(' Network IP allocation test passed')
}
// Test 7: IPv4 connectivity test with real Alpine container
fn test_ipv4_connectivity() ! {
skip_if_not_linux()
test_name := 'test_ipv4_${os.getpid()}'
defer {
cleanup_test_heropods(test_name)
}
mut hp := new(
name: test_name
reset: true
use_podman: true
)!
container_name := 'ipv4_${os.getpid()}'
mut container := hp.container_new(
name: container_name
image: .custom
custom_image_name: 'alpine_ipv4'
docker_url: 'docker.io/library/alpine:3.20'
)!
container.start()!
defer {
container.stop() or {}
container.delete() or {}
}
// Check container has an IP address
container_ip := hp.network_config.allocated_ips[container_name] or {
return error('Container should have allocated IP')
}
assert container_ip.starts_with('10.10.0.')
// Test IPv4 connectivity by checking the container's IP configuration
result := container.exec(cmd: 'ip addr show eth0')!
assert result.contains(container_ip)
assert result.contains('eth0')
// Test that default route exists
route_result := container.exec(cmd: 'ip route')!
assert route_result.contains('default')
assert route_result.contains('10.10.0.1')
println(' IPv4 connectivity test passed')
}
// Test 8: Container deletion and IP cleanup
fn test_container_deletion() ! {
skip_if_not_linux()
test_name := 'test_delete_${os.getpid()}'
defer {
cleanup_test_heropods(test_name)
}
mut hp := new(
name: test_name
reset: true
use_podman: true
)!
container_name := 'delete_${os.getpid()}'
mut container := hp.container_new(
name: container_name
image: .custom
custom_image_name: 'alpine_delete'
docker_url: 'docker.io/library/alpine:3.20'
)!
// Start container (allocates IP)
container.start()!
// Verify IP is allocated
assert container_name in hp.network_config.allocated_ips
// Stop and delete container
container.stop()!
container.delete()!
// Verify container is deleted from crun
exists := container.container_exists_in_crun()!
assert !exists
// Verify IP is freed
assert container_name !in hp.network_config.allocated_ips
println(' Container deletion and IP cleanup test passed')
}

View File

@@ -0,0 +1,278 @@
module heropods
import incubaid.herolib.core
import incubaid.herolib.osal.core as osal
import os
// Network-specific tests for HeroPods
//
// These tests verify bridge setup, IP allocation, NAT rules, and network cleanup
// Helper function to check if we're on Linux
fn is_linux_platform() bool {
return core.is_linux() or { false }
}
// Helper function to skip test if not on Linux
fn skip_if_not_linux() {
if !is_linux_platform() {
eprintln('SKIP: Test requires Linux (crun, ip, iptables)')
exit(0)
}
}
// Setup minimal test rootfs for testing
fn setup_test_rootfs() ! {
rootfs_path := os.home_dir() + '/.containers/images/alpine/rootfs'
// Skip if already exists and has valid binaries
if os.is_dir(rootfs_path) && os.is_file('${rootfs_path}/bin/sh') {
// Check if sh is a real binary (> 1KB)
sh_info := os.stat('${rootfs_path}/bin/sh') or { return }
if sh_info.size > 1024 {
return
}
}
// Remove old rootfs if it exists
if os.is_dir(rootfs_path) {
os.rmdir_all(rootfs_path) or {}
}
// Create minimal rootfs structure
os.mkdir_all(rootfs_path)!
os.mkdir_all('${rootfs_path}/bin')!
os.mkdir_all('${rootfs_path}/etc')!
os.mkdir_all('${rootfs_path}/dev')!
os.mkdir_all('${rootfs_path}/proc')!
os.mkdir_all('${rootfs_path}/sys')!
os.mkdir_all('${rootfs_path}/tmp')!
os.mkdir_all('${rootfs_path}/usr/bin')!
os.mkdir_all('${rootfs_path}/usr/local/bin')!
os.mkdir_all('${rootfs_path}/lib/x86_64-linux-gnu')!
os.mkdir_all('${rootfs_path}/lib64')!
// Copy essential binaries from host
// Use dash (smaller than bash) and sleep
if os.exists('/bin/dash') {
os.execute('cp -L /bin/dash ${rootfs_path}/bin/sh')
os.chmod('${rootfs_path}/bin/sh', 0o755)!
} else if os.exists('/bin/sh') {
os.execute('cp -L /bin/sh ${rootfs_path}/bin/sh')
os.chmod('${rootfs_path}/bin/sh', 0o755)!
}
// Copy common utilities
for cmd in ['sleep', 'echo', 'cat', 'ls', 'pwd', 'true', 'false'] {
if os.exists('/bin/${cmd}') {
os.execute('cp -L /bin/${cmd} ${rootfs_path}/bin/${cmd}')
os.chmod('${rootfs_path}/bin/${cmd}', 0o755) or {}
} else if os.exists('/usr/bin/${cmd}') {
os.execute('cp -L /usr/bin/${cmd} ${rootfs_path}/bin/${cmd}')
os.chmod('${rootfs_path}/bin/${cmd}', 0o755) or {}
}
}
// Copy required libraries for dash/sh
// Copy from /lib/x86_64-linux-gnu to the same path in rootfs
if os.is_dir('/lib/x86_64-linux-gnu') {
os.execute('cp -a /lib/x86_64-linux-gnu/libc.so.6 ${rootfs_path}/lib/x86_64-linux-gnu/')
os.execute('cp -a /lib/x86_64-linux-gnu/libc-*.so ${rootfs_path}/lib/x86_64-linux-gnu/ 2>/dev/null || true')
// Copy dynamic linker (actual file, not symlink)
os.execute('cp -L /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 ${rootfs_path}/lib/x86_64-linux-gnu/')
}
// Create symlink in /lib64 pointing to the actual file
if os.is_dir('${rootfs_path}/lib64') {
os.execute('ln -sf ../lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 ${rootfs_path}/lib64/ld-linux-x86-64.so.2')
}
// Create /etc/resolv.conf
os.write_file('${rootfs_path}/etc/resolv.conf', 'nameserver 8.8.8.8\n')!
}
// Cleanup helper for tests
fn cleanup_test_heropods(name string) {
mut hp := get(name: name) or { return }
for container_name, mut container in hp.containers {
container.stop() or {}
container.delete() or {}
}
// Don't delete the bridge (false) - tests run in parallel and share the same bridge
// Only clean up containers and IPs
hp.network_cleanup_all(false) or {}
delete(name: name) or {}
}
// Test 1: Bridge network setup
fn test_network_bridge_setup() ! {
skip_if_not_linux()
test_name := 'test_bridge_${os.getpid()}'
mut hp := new(
name: test_name
reset: true
use_podman: true // Skip default image setup in tests
)!
bridge_name := hp.network_config.bridge_name
// Verify bridge exists
job := osal.exec(cmd: 'ip link show ${bridge_name}')!
assert job.output.contains(bridge_name)
// Verify bridge is UP
assert job.output.contains('UP') || job.output.contains('state UP')
// Verify IP is assigned to bridge
job2 := osal.exec(cmd: 'ip addr show ${bridge_name}')!
assert job2.output.contains(hp.network_config.gateway_ip)
// Cleanup after test
cleanup_test_heropods(test_name)
println(' Bridge network setup test passed')
}
// Test 2: NAT rules verification
fn test_network_nat_rules() ! {
skip_if_not_linux()
test_name := 'test_nat_${os.getpid()}'
defer { cleanup_test_heropods(test_name) }
mut hp := new(
name: test_name
reset: true
use_podman: true // Skip default image setup in tests
)!
// Verify NAT rules exist for the subnet
job := osal.exec(cmd: 'iptables -t nat -L POSTROUTING -n')!
assert job.output.contains(hp.network_config.subnet) || job.output.contains('MASQUERADE')
println(' NAT rules test passed')
}
// Test 3: IP allocation sequential
fn test_ip_allocation_sequential() ! {
skip_if_not_linux()
test_name := 'test_ip_seq_${os.getpid()}'
defer { cleanup_test_heropods(test_name) }
mut hp := new(
name: test_name
reset: true
use_podman: true // Skip default image setup in tests
)!
// Allocate multiple IPs
mut allocated_ips := []string{}
for i in 0 .. 10 {
ip := hp.network_allocate_ip('container_${i}')!
allocated_ips << ip
}
// Verify all IPs are unique
for i, ip1 in allocated_ips {
for j, ip2 in allocated_ips {
if i != j {
assert ip1 != ip2, 'IPs should be unique: ${ip1} == ${ip2}'
}
}
}
// Verify all IPs are in correct subnet
for ip in allocated_ips {
assert ip.starts_with('10.10.0.')
}
println(' IP allocation sequential test passed')
}
// Test 4: IP pool management with container lifecycle
fn test_ip_pool_management() ! {
skip_if_not_linux()
setup_test_rootfs()!
test_name := 'test_ip_pool_${os.getpid()}'
defer { cleanup_test_heropods(test_name) }
mut hp := new(
name: test_name
reset: true
use_podman: true // Skip default image setup in tests
)!
// Create and start 3 containers
mut container1 := hp.container_new(name: 'pool_test1_${os.getpid()}', image: .alpine_3_20)!
mut container2 := hp.container_new(name: 'pool_test2_${os.getpid()}', image: .alpine_3_20)!
mut container3 := hp.container_new(name: 'pool_test3_${os.getpid()}', image: .alpine_3_20)!
container1.start()!
container2.start()!
container3.start()!
// Get allocated IPs
ip1 := hp.network_config.allocated_ips[container1.name]
ip2 := hp.network_config.allocated_ips[container2.name]
ip3 := hp.network_config.allocated_ips[container3.name]
// Delete middle container (frees IP2)
container2.stop()!
container2.delete()!
// Verify IP2 is freed
assert container2.name !in hp.network_config.allocated_ips
// Create new container - should reuse freed IP2
mut container4 := hp.container_new(name: 'pool_test4_${os.getpid()}', image: .alpine_3_20)!
container4.start()!
ip4 := hp.network_config.allocated_ips[container4.name]
assert ip4 == ip2, 'Should reuse freed IP: ${ip2} vs ${ip4}'
// Cleanup
container1.stop()!
container1.delete()!
container3.stop()!
container3.delete()!
container4.stop()!
container4.delete()!
println(' IP pool management test passed')
}
// Test 5: Custom bridge configuration
fn test_custom_bridge_config() ! {
skip_if_not_linux()
test_name := 'test_custom_br_${os.getpid()}'
custom_bridge := 'custombr_${os.getpid()}'
defer {
cleanup_test_heropods(test_name)
// Cleanup custom bridge
osal.exec(cmd: 'ip link delete ${custom_bridge}') or {}
}
mut hp := new(
name: test_name
reset: true
use_podman: true // Skip default image setup in tests
bridge_name: custom_bridge
subnet: '172.20.0.0/24'
gateway_ip: '172.20.0.1'
)!
// Verify custom bridge exists
job := osal.exec(cmd: 'ip link show ${custom_bridge}')!
assert job.output.contains(custom_bridge)
// Verify custom IP
job2 := osal.exec(cmd: 'ip addr show ${custom_bridge}')!
assert job2.output.contains('172.20.0.1')
println(' Custom bridge configuration test passed')
}