herolib/lib/virt/hetznermanager/rescue.v

module hetznermanager

import incubaid.herolib.core.texttools
import time
import incubaid.herolib.ui.console
import incubaid.herolib.osal.core as osal
import incubaid.herolib.builder
import os

// ///////////////////////////RESCUE

pub struct RescueInfo {
pub mut:
	server_ip       string
	server_ipv6_net string
	server_number   int
	os              string
	arch            int
	active          bool
	password        string
	authorized_key  []string
	host_key        []string
}

pub struct ServerRescueArgs {
pub mut:
	id           int
	name         string
	wait         bool = true
	hero_install bool
	reset        bool // ask to do reset/rescue even if its already in that state
	retry        int = 3
}

pub fn (mut h HetznerManager) server_rescue(args_ ServerRescueArgs) !ServerInfoDetailed {
	h.check_whitelist(args_)!
	if args_.retry > 1 {
		for _ in 0 .. args_.retry - 1 {
			return h.server_rescue_internal(args_) or { continue }
		}
		console.print_header('server ${args_.name} failed to rescue we retry: now ${args_.retry} attempts')
	}
	return h.server_rescue_internal(args_)!
}

fn (mut h HetznerManager) server_rescue_internal(args_ ServerRescueArgs) !ServerInfoDetailed {
	mut args := args_
	mut serverinfo := h.server_info_get(id: args.id, name: args.name)!

	os.execute_opt('ssh-keygen -R ${serverinfo.server_ip}')!

	if serverinfo.rescue && !args.reset {
		if osal.ssh_test(address: serverinfo.server_ip, port: 22)! == .ok {
			console.print_debug('test server ${serverinfo.server_name} is in rescue mode?')
			mut b := builder.new()!
			mut n := b.node_new(ipaddr: serverinfo.server_ip)!

			res := n.exec(cmd: 'ls /root/.oldroot/nfs/install/installimage', stdout: false) or {
				'ERROR'
			}
			if res.contains('nfs/install/installimage') {
				console.print_debug('server ${serverinfo.server_name} is in rescue mode')
				return serverinfo
			}
		}
		serverinfo.rescue = false
	}
	// only do it if its not in rescue yet
	if serverinfo.rescue == false || args.reset {
		console.print_header('server ${serverinfo.server_name} goes into rescue mode')

		mykey := h.key_get(h.sshkey)!
		mykeyfp := mykey.fingerprint

		// println("Using SSH key fingerprint: ${mykey} ${mykeyfp}")

		mut conn := h.connection()!
		rescue := conn.post_json_generic[RescueInfo](
			prefix:     'boot/${serverinfo.server_number}/rescue'
			params:     {
				'os':             'linux'
				'authorized_key': mykeyfp
			}
			dict_key:   'rescue'
			dataformat: .urlencoded
		)!

		console.print_debug('Request for hetzner rescue done.\n${rescue}')

		h.server_reset(
			id:   args.id
			name: args.name
			wait: args.wait
			msg:  ' to get up and running in rescue mode.'
		)!

		os.execute_opt('ssh-keygen -R ${serverinfo.server_ip}')!
	}

	if args.hero_install {
		args.wait = true
	}

	if args.wait {
		mut b := builder.new()!
		mut n := b.node_new(ipaddr: serverinfo.server_ip)!
		n.exec_silent('apt update && apt install -y mc redis')!
		if args.hero_install {
			n.hero_install()!
		}
	}

	mut serverinfo2 := h.server_info_get(id: args.id, name: args.name)!

	return serverinfo2
}

pub fn (mut h HetznerManager) server_rescue_node(args ServerRescueArgs) !&builder.Node {
	mut serverinfo := h.server_rescue(args)!

	mut b := builder.new()!
	mut n := b.node_new(ipaddr: serverinfo.server_ip)!

	return n
}

pub struct ServerInstallArgs {
pub mut:
	id                   int
	name                 string
	wait                 bool = true
	hero_install         bool
	hero_install_compile bool
	raid                 bool
}

pub fn (mut h HetznerManager) ubuntu_install(args ServerInstallArgs) !&builder.Node {
	h.check_whitelist(name: args.name, id: args.id)!
	mut serverinfo := h.server_rescue(
		id:   args.id
		name: args.name
		wait: true
	)!

	mut b := builder.new()!
	mut n := b.node_new(ipaddr: serverinfo.server_ip)!

	// installconfig:=$tmpl("templates/ubuntu_install.sh")
	// n.file_write("/tmp/installconfig",installconfig)!
	// n.exec_interactive("installimage -a -c /tmp/installconfig")!

	mut rstr := '-r no '
	if args.raid {
		panic('should not use RAID for now')
		rstr = '-r yes -l 1 '
	}

	n.exec(
		cmd: '
		set -ex
		echo "go into install mode, try to install ubuntu 24.04"

		if [ -d /sys/firmware/efi ]; then
			echo "UEFI system detected → need ESP"
			PARTS="/boot/efi:esp:256M,swap:swap:4G,/boot:ext3:1024M,/:btrfs:all"
		else
			echo "BIOS/legacy system detected → no ESP"
			PARTS="swap:swap:4G,/boot:ext3:1024M,/:btrfs:all"
		fi

		# installimage invocation
		/root/.oldroot/nfs/install/installimage -a -n "${args.name}" ${rstr} -i /root/.oldroot/nfs/images/Ubuntu-2404-noble-amd64-base.tar.gz -f yes -t yes -p "\$PARTS"

		reboot
		'
	)!

	os.execute_opt('ssh-keygen -R ${serverinfo.server_ip}')!

	console.print_debug('server ${serverinfo.server_name} is installed in ubuntu now, should be restarting.')

	osal.reboot_wait(
		address:      serverinfo.server_ip
		timeout_down: 60
		timeout_up:   60 * 5
	)!

	console.print_debug('server ${serverinfo.server_name} is reacheable over ping, lets now try ssh.')

	// wait 20 sec to make sure ssh is there
	osal.ssh_wait(address: serverinfo.server_ip, timeout: 20)!

	console.print_debug('server ${serverinfo.server_name} is reacheable over ssh, lets now install hero if asked for.')

	if args.hero_install {
		n.exec_silent('apt update && apt install -y mc redis libpq5 libpq-dev')!
		n.hero_install(compile: args.hero_install_compile)!
	}

	return n
}