MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

feat: Add custom crun root and enhance container lifecycle

Browse Source 
- Use custom `crun --root` for all container commands
- Implement `cleanup_crun_state` for factory reset
- Add retry logic for `crun create` on "File exists" error
- Improve OCI config with `set_terminal`, unique env/rlimits
- Add default mounts for `/dev/pts`, `/dev/shm`, `/dev/mqueue`, `/sys/fs/cgroup`
This commit is contained in:
Mahmoud-Emad
2025-09-10 13:44:32 +03:00
parent 2ddec79102
commit 6c971ca689
9 changed files with 219 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/virt/crun/example.v
View File

@@ -5,7 +5,8 @@ pub fn example_heropods_compatible() ! {
// Create a container configuration compatible with heropods template
mut config := new(mut configs, name: 'heropods-example')!
// Configure to match the template
// Configure to match the template - disable terminal for background containers
config.set_terminal(false)
config.set_command(['/bin/sh'])
config.set_working_dir('/')
config.set_user(0, 0, [])

158
lib/virt/crun/factory.v
View File

@@ -2,11 +2,10 @@ module crun
import freeflowuniverse.herolib.core.texttools
@[params]
pub struct FactoryArgs {
pub mut:
name string = "default"
name string = 'default'
}
pub struct CrunConfig {
@@ -23,6 +22,8 @@ pub fn (mount_type MountType) to_string() string {
.proc { 'proc' }
.sysfs { 'sysfs' }
.devpts { 'devpts' }
.mqueue { 'mqueue' }
.cgroup { 'cgroup' }
.nfs { 'nfs' }
.overlay { 'overlay' }
}
@@ -120,21 +121,23 @@ pub fn (mut config CrunConfig) set_working_dir(cwd string) &CrunConfig {
pub fn (mut config CrunConfig) set_user(uid u32, gid u32, additional_gids []u32) &CrunConfig {
config.spec.process.user = User{
uid: uid
gid: gid
uid: uid
gid: gid
additional_gids: additional_gids.clone()
}
return config
}
pub fn (mut config CrunConfig) add_env(key string, value string) &CrunConfig {
// Remove existing env var with same key to avoid duplicates
config.spec.process.env = config.spec.process.env.filter(!it.starts_with('${key}='))
config.spec.process.env << '${key}=${value}'
return config
}
pub fn (mut config CrunConfig) set_rootfs(path string, readonly bool) &CrunConfig {
config.spec.root = Root{
path: path
path: path
readonly: readonly
}
return config
@@ -165,16 +168,16 @@ pub fn (mut config CrunConfig) set_pids_limit(limit i64) &CrunConfig {
pub fn (mut config CrunConfig) add_mount(destination string, source string, typ MountType, options []MountOption) &CrunConfig {
config.spec.mounts << Mount{
destination: destination
typ: typ.to_string()
source: source
options: options.map(it.to_string())
typ: typ.to_string()
source: source
options: options.map(it.to_string())
}
return config
}
pub fn (mut config CrunConfig) add_capability(cap Capability) &CrunConfig {
cap_str := cap.to_string()
if cap_str !in config.spec.process.capabilities.bounding {
config.spec.process.capabilities.bounding << cap_str
}
@@ -189,7 +192,7 @@ pub fn (mut config CrunConfig) add_capability(cap Capability) &CrunConfig {
pub fn (mut config CrunConfig) remove_capability(cap Capability) &CrunConfig {
cap_str := cap.to_string()
config.spec.process.capabilities.bounding = config.spec.process.capabilities.bounding.filter(it != cap_str)
config.spec.process.capabilities.effective = config.spec.process.capabilities.effective.filter(it != cap_str)
config.spec.process.capabilities.permitted = config.spec.process.capabilities.permitted.filter(it != cap_str)
@@ -197,8 +200,11 @@ pub fn (mut config CrunConfig) remove_capability(cap Capability) &CrunConfig {
}
pub fn (mut config CrunConfig) add_rlimit(typ RlimitType, hard u64, soft u64) &CrunConfig {
// Remove existing rlimit with same type to avoid duplicates
typ_str := typ.to_string()
config.spec.process.rlimits = config.spec.process.rlimits.filter(it.typ != typ_str)
config.spec.process.rlimits << Rlimit{
typ: typ.to_string()
typ: typ_str
hard: hard
soft: soft
}
@@ -210,6 +216,11 @@ pub fn (mut config CrunConfig) set_no_new_privileges(value bool) &CrunConfig {
return config
}
pub fn (mut config CrunConfig) set_terminal(value bool) &CrunConfig {
config.spec.process.terminal = value
return config
}
pub fn (mut config CrunConfig) add_masked_path(path string) &CrunConfig {
if path !in config.spec.linux.masked_paths {
config.spec.linux.masked_paths << path
@@ -226,67 +237,65 @@ pub fn (mut config CrunConfig) add_readonly_path(path string) &CrunConfig {
pub fn new(mut configs map[string]&CrunConfig, args FactoryArgs) !&CrunConfig {
name := texttools.name_fix(args.name)
mut config := &CrunConfig{
name: name
spec: create_default_spec()
}
configs[name] = config
return config
}
pub fn get(configs map[string]&CrunConfig, args FactoryArgs) !&CrunConfig {
name := texttools.name_fix(args.name)
return configs[name] or {
return error('crun config with name "${name}" does not exist')
}
return configs[name] or { return error('crun config with name "${name}" does not exist') }
}
fn create_default_spec() Spec {
// Create default spec that matches the heropods template
mut spec := Spec{
oci_version: '1.0.2' // Set default here
platform: Platform{
os: 'linux'
platform: Platform{
os: 'linux'
arch: 'amd64'
}
process: Process{
terminal: true
user: User{
process: Process{
terminal: true
user: User{
uid: 0
gid: 0
}
args: ['/bin/sh']
env: [
args: ['/bin/sh']
env: [
'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',
'TERM=xterm'
'TERM=xterm',
]
cwd: '/'
capabilities: Capabilities{
bounding: ['CAP_AUDIT_WRITE', 'CAP_KILL', 'CAP_NET_BIND_SERVICE']
effective: ['CAP_AUDIT_WRITE', 'CAP_KILL', 'CAP_NET_BIND_SERVICE']
cwd: '/'
capabilities: Capabilities{
bounding: ['CAP_AUDIT_WRITE', 'CAP_KILL', 'CAP_NET_BIND_SERVICE']
effective: ['CAP_AUDIT_WRITE', 'CAP_KILL', 'CAP_NET_BIND_SERVICE']
inheritable: ['CAP_AUDIT_WRITE', 'CAP_KILL', 'CAP_NET_BIND_SERVICE']
permitted: ['CAP_AUDIT_WRITE', 'CAP_KILL', 'CAP_NET_BIND_SERVICE']
permitted: ['CAP_AUDIT_WRITE', 'CAP_KILL', 'CAP_NET_BIND_SERVICE']
}
rlimits: [
rlimits: [
Rlimit{
typ: 'RLIMIT_NOFILE'
typ: 'RLIMIT_NOFILE'
hard: 1024
soft: 1024
}
},
]
no_new_privileges: true // No JSON annotation needed here
}
root: Root{
path: 'rootfs'
root: Root{
path: 'rootfs'
readonly: false
}
hostname: 'container'
mounts: create_default_mounts()
linux: Linux{
namespaces: create_default_namespaces()
masked_paths: [
hostname: 'container'
mounts: create_default_mounts()
linux: Linux{
namespaces: create_default_namespaces()
masked_paths: [
'/proc/acpi',
'/proc/kcore',
'/proc/keys',
@@ -295,7 +304,7 @@ fn create_default_spec() Spec {
'/proc/timer_stats',
'/proc/sched_debug',
'/proc/scsi',
'/sys/firmware'
'/sys/firmware',
]
readonly_paths: [
'/proc/asound',
@@ -303,21 +312,34 @@ fn create_default_spec() Spec {
'/proc/fs',
'/proc/irq',
'/proc/sys',
'/proc/sysrq-trigger'
'/proc/sysrq-trigger',
]
}
}
return spec
}
fn create_default_namespaces() []LinuxNamespace {
return [
LinuxNamespace{typ: 'pid'},
LinuxNamespace{typ: 'network'},
LinuxNamespace{typ: 'ipc'},
LinuxNamespace{typ: 'uts'},
LinuxNamespace{typ: 'mount'},
LinuxNamespace{
typ: 'pid'
},
LinuxNamespace{
typ: 'network'
},
LinuxNamespace{
typ: 'ipc'
},
LinuxNamespace{
typ: 'uts'
},
LinuxNamespace{
typ: 'cgroup'
},
LinuxNamespace{
typ: 'mount'
},
]
}
@@ -325,20 +347,44 @@ fn create_default_mounts() []Mount {
return [
Mount{
destination: '/proc'
typ: 'proc'
source: 'proc'
typ: 'proc'
source: 'proc'
},
Mount{
destination: '/dev'
typ: 'tmpfs'
source: 'tmpfs'
options: ['nosuid', 'strictatime', 'mode=755', 'size=65536k']
typ: 'tmpfs'
source: 'tmpfs'
options: ['nosuid', 'strictatime', 'mode=755', 'size=65536k']
},
Mount{
destination: '/dev/pts'
typ: 'devpts'
source: 'devpts'
options: ['nosuid', 'noexec', 'newinstance', 'ptmxmode=0666', 'mode=0620', 'gid=5']
},
Mount{
destination: '/dev/shm'
typ: 'tmpfs'
source: 'shm'
options: ['nosuid', 'noexec', 'nodev', 'mode=1777', 'size=65536k']
},
Mount{
destination: '/dev/mqueue'
typ: 'mqueue'
source: 'mqueue'
options: ['nosuid', 'noexec', 'nodev']
},
Mount{
destination: '/sys'
typ: 'sysfs'
source: 'sysfs'
options: ['nosuid', 'noexec', 'nodev', 'ro']
typ: 'sysfs'
source: 'sysfs'
options: ['nosuid', 'noexec', 'nodev', 'ro']
},
Mount{
destination: '/sys/fs/cgroup'
typ: 'cgroup'
source: 'cgroup'
options: ['nosuid', 'noexec', 'nodev', 'relatime', 'ro']
},
]
}
}

28
lib/virt/crun/model.v
View File

@@ -82,7 +82,7 @@ pub mut:
pub struct LinuxNamespace {
pub mut:
typ string @[json: 'type']
path string
path string @[omitempty]
}
pub struct LinuxResources {
@@ -95,30 +95,30 @@ pub mut:
pub struct Memory {
pub mut:
limit u64
reservation u64
swap u64
kernel u64
swappiness i64
limit u64 @[omitempty]
reservation u64 @[omitempty]
swap u64 @[omitempty]
kernel u64 @[omitempty]
swappiness i64 @[omitempty]
}
pub struct CPU {
pub mut:
shares u64
quota i64
period u64
cpus string
mems string
shares u64 @[omitempty]
quota i64 @[omitempty]
period u64 @[omitempty]
cpus string @[omitempty]
mems string @[omitempty]
}
pub struct Pids {
pub mut:
limit i64
limit i64 @[omitempty]
}
pub struct BlockIO {
pub mut:
weight u16
weight u16 @[omitempty]
}
pub struct LinuxDevice {
@@ -160,6 +160,8 @@ pub enum MountType {
proc
sysfs
devpts
mqueue
cgroup
nfs
overlay
}

58
lib/virt/heropods/container.v
View File

@@ -34,10 +34,32 @@ pub fn (mut self Container) start() ! {
if !container_exists {
// Container doesn't exist, create it first
console.print_debug('Container ${self.name} does not exist, creating it...')
osal.exec(
cmd: 'crun create --bundle ${self.factory.base_dir}/configs/${self.name} ${self.name}'
// Try to create the container, if it fails with "File exists" error,
// try to force delete any leftover state and retry
crun_root := '${self.factory.base_dir}/runtime'
create_result := osal.exec(
cmd: 'crun --root ${crun_root} create --bundle ${self.factory.base_dir}/configs/${self.name} ${self.name}'
stdout: true
)!
) or {
if err.msg().contains('File exists') {
console.print_debug('Container creation failed with "File exists", attempting to clean up leftover state...')
// Force delete any leftover state - try multiple cleanup approaches
osal.exec(cmd: 'crun --root ${crun_root} delete ${self.name}', stdout: false) or {}
osal.exec(cmd: 'crun delete ${self.name}', stdout: false) or {} // Also try default root
// Clean up any leftover runtime directories
osal.exec(cmd: 'rm -rf ${crun_root}/${self.name}', stdout: false) or {}
osal.exec(cmd: 'rm -rf /run/crun/${self.name}', stdout: false) or {}
// Wait a moment for cleanup to complete
time.sleep(500 * time.millisecond)
// Retry creation
osal.exec(
cmd: 'crun --root ${crun_root} create --bundle ${self.factory.base_dir}/configs/${self.name} ${self.name}'
stdout: true
)!
} else {
return err
}
}
console.print_debug('Container ${self.name} created')
}
@@ -51,16 +73,18 @@ pub fn (mut self Container) start() ! {
// because crun doesn't allow restarting a stopped container
if container_exists && status != .running {
console.print_debug('Container ${self.name} exists but is stopped, recreating...')
osal.exec(cmd: 'crun delete ${self.name}', stdout: false) or {}
crun_root := '${self.factory.base_dir}/runtime'
osal.exec(cmd: 'crun --root ${crun_root} delete ${self.name}', stdout: false) or {}
osal.exec(
cmd: 'crun create --bundle ${self.factory.base_dir}/configs/${self.name} ${self.name}'
cmd: 'crun --root ${crun_root} create --bundle ${self.factory.base_dir}/configs/${self.name} ${self.name}'
stdout: true
)!
console.print_debug('Container ${self.name} recreated')
}
// start the container (crun start doesn't have --detach flag)
osal.exec(cmd: 'crun start ${self.name}', stdout: true)!
crun_root := '${self.factory.base_dir}/runtime'
osal.exec(cmd: 'crun --root ${crun_root} start ${self.name}', stdout: true)!
console.print_green('Container ${self.name} started')
}
@@ -71,12 +95,13 @@ pub fn (mut self Container) stop() ! {
return
}
osal.exec(cmd: 'crun kill ${self.name} SIGTERM', stdout: false) or {}
crun_root := '${self.factory.base_dir}/runtime'
osal.exec(cmd: 'crun --root ${crun_root} kill ${self.name} SIGTERM', stdout: false) or {}
time.sleep(2 * time.second)
// Force kill if still running
if self.status()! == .running {
osal.exec(cmd: 'crun kill ${self.name} SIGKILL', stdout: false) or {}
osal.exec(cmd: 'crun --root ${crun_root} kill ${self.name} SIGKILL', stdout: false) or {}
}
console.print_green('Container ${self.name} stopped')
}
@@ -89,7 +114,8 @@ pub fn (mut self Container) delete() ! {
}
self.stop()!
osal.exec(cmd: 'crun delete ${self.name}', stdout: false) or {}
crun_root := '${self.factory.base_dir}/runtime'
osal.exec(cmd: 'crun --root ${crun_root} delete ${self.name}', stdout: false) or {}
// Remove from factory's container cache
if self.name in self.factory.containers {
@@ -113,7 +139,10 @@ pub fn (mut self Container) exec(cmd_ osal.Command) !string {
}
pub fn (self Container) status() !ContainerStatus {
result := osal.exec(cmd: 'crun state ${self.name}', stdout: false) or { return .unknown }
crun_root := '${self.factory.base_dir}/runtime'
result := osal.exec(cmd: 'crun --root ${crun_root} state ${self.name}', stdout: false) or {
return .unknown
}
// Parse JSON output from crun state
state := json.decode(CrunState, result.output) or { return .unknown }
@@ -129,7 +158,10 @@ pub fn (self Container) status() !ContainerStatus {
// Check if container exists in crun (regardless of its state)
fn (self Container) container_exists_in_crun() !bool {
// Try to get container state - if it fails, container doesn't exist
result := osal.exec(cmd: 'crun state ${self.name}', stdout: false) or { return false }
crun_root := '${self.factory.base_dir}/runtime'
result := osal.exec(cmd: 'crun --root ${crun_root} state ${self.name}', stdout: false) or {
return false
}
// If we get here, the container exists (even if stopped/paused)
return result.exit_code == 0
@@ -209,7 +241,8 @@ pub fn (mut self Container) tmux_pane(args TmuxPaneArgs) !&tmux.Pane {
// Execute command if provided
if args.cmd != '' {
pane.send_keys('crun exec ${self.name} ${args.cmd}')!
crun_root := '${self.factory.base_dir}/runtime'
pane.send_keys('crun --root ${crun_root} exec ${self.name} ${args.cmd}')!
}
self.tmux_pane = pane
@@ -226,6 +259,7 @@ pub fn (mut self Container) node() !&builder.Node {
mut exec := builder.ExecutorCrun{
container_id: self.name
crun_root: '${self.factory.base_dir}/runtime'
debug: false
}

1
lib/virt/heropods/container_create.v
View File

@@ -93,6 +93,7 @@ fn (mut self ContainerFactory) create_crun_config(container_name string, rootfs_
mut config := crun.new(mut self.crun_configs, name: container_name)!
// Configure for heropods use case - disable terminal for background containers
config.set_terminal(false)
config.set_command(['/bin/sh', '-c', 'while true; do sleep 30; done'])
config.set_working_dir('/')
config.set_user(0, 0, [])

36
lib/virt/heropods/factory.v
View File

@@ -46,6 +46,11 @@ fn (mut self ContainerFactory) init(args FactoryInitArgs) ! {
}
}
// Clean up any leftover crun state if reset is requested
if args.reset {
self.cleanup_crun_state()!
}
// Load existing images into cache
self.load_existing_images()!
@@ -137,3 +142,34 @@ pub fn (self ContainerFactory) list() ![]Container {
}
return containers
}
// Clean up any leftover crun state
fn (mut self ContainerFactory) cleanup_crun_state() ! {
console.print_debug('Cleaning up leftover crun state...')
crun_root := '${self.base_dir}/runtime'
// Stop and delete all containers in our custom root
result := osal.exec(cmd: 'crun --root ${crun_root} list -q', stdout: false) or { return }
for container_name in result.output.split_into_lines() {
if container_name.trim_space() != '' {
console.print_debug('Cleaning up container: ${container_name}')
osal.exec(cmd: 'crun --root ${crun_root} kill ${container_name} SIGKILL', stdout: false) or {}
osal.exec(cmd: 'crun --root ${crun_root} delete ${container_name}', stdout: false) or {}
}
}
// Also clean up any containers in the default root that might be ours
result2 := osal.exec(cmd: 'crun list -q', stdout: false) or { return }
for container_name in result2.output.split_into_lines() {
if container_name.trim_space() != '' && container_name in self.containers {
console.print_debug('Cleaning up container from default root: ${container_name}')
osal.exec(cmd: 'crun kill ${container_name} SIGKILL', stdout: false) or {}
osal.exec(cmd: 'crun delete ${container_name}', stdout: false) or {}
}
}
// Clean up runtime directories
osal.exec(cmd: 'rm -rf ${crun_root}/*', stdout: false) or {}
osal.exec(cmd: 'find /run/crun -name "*" -type d -exec rm -rf {} + 2>/dev/null', stdout: false) or {}
}