zosbuilder/scripts/lib/components.sh

#!/bin/bash
# Component download and build system for ThreeFold Zero OS

# Source common functions
LIB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${LIB_SCRIPT_DIR}/common.sh"

# Component configuration
RUST_TARGET="${RUST_TARGET:-x86_64-unknown-linux-musl}"
CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}"

# Parse and process all components from sources.conf
function components_parse_sources_conf() {
	local sources_file="$1"
	local components_dir="$2"
	local install_dir="${INSTALL_DIR:-${PROJECT_ROOT}/initramfs}"

	section_header "Parsing Sources Configuration"

	if [[ ! -f "$sources_file" ]]; then
		log_error "Sources file not found: ${sources_file}"
		return 1
	fi

	# Ensure components directory exists
	safe_mkdir "$components_dir"

	# Export install directory for build functions
	export INSTALL_DIR="$install_dir"

	log_info "Processing components from: ${sources_file}"
	log_info "Components directory: ${components_dir}"
	log_info "Install directory: ${install_dir}"

	local component_count=0

	# Read entries from sources.conf (TYPE NAME URL VERSION BUILD_FUNCTION [EXTRA])
	while IFS= read -r _raw || [[ -n "$_raw" ]]; do
		# Strip comments and trim whitespace
		local line="${_raw%%#*}"
		line="${line#"${line%%[![:space:]]*}"}"
		line="${line%"${line##*[![:space:]]}"}"
		[[ -z "$line" ]] && continue

		local type name url version build_func extra
		# shellcheck disable=SC2086
		read -r type name url version build_func extra <<<"$line"

		if [[ -z "${type:-}" || -z "${name:-}" || -z "${url:-}" || -z "${version:-}" || -z "${build_func:-}" ]]; then
			log_warn "Skipping malformed entry: ${_raw}"
			continue
		fi

		component_count=$((component_count + 1))
		log_info "Processing component ${component_count}: ${name} (${type})"

		case "$type" in
		git)
			components_download_git "$name" "$url" "$version" "$components_dir"
			;;
		release)
			components_download_release "$name" "$url" "$version" "$components_dir" "$extra"
			;;
		*)
			log_error "Unknown component type in sources.conf: ${type}"
			return 1
			;;
		esac

		components_build_component "$name" "$build_func" "$components_dir"
	done <"$sources_file"

	if [[ $component_count -eq 0 ]]; then
		log_warn "No components found in sources configuration"
	else
		log_info "Processed ${component_count} components successfully"
	fi
}

# Download Git repository (reuse tree; only reclone if invalid or version not reachable)
function components_download_git() {
	local name="$1"
	local url="$2"
	local version="$3"
	local components_dir="$4"

	section_header "Downloading Git Component: ${name}"

	local target_dir="${components_dir}/${name}"

	log_info "Repository: ${url}"
	log_info "Version/Branch/Tag: ${version}"
	log_info "Target directory: ${target_dir}"

	# Ensure parent exists
	safe_mkdir "$components_dir"

	# Decide whether we can reuse the existing working tree
	local need_fresh_clone="0"
	if [[ -d "$target_dir/.git" ]]; then
		if ! git -C "$target_dir" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
			log_warn "Existing ${name} directory is not a valid git repo; will reclone"
			need_fresh_clone="1"
		fi
	elif [[ -d "$target_dir" ]]; then
		log_warn "Existing ${name} directory without .git; will reclone"
		need_fresh_clone="1"
	fi

	if [[ "$need_fresh_clone" == "1" || ! -d "$target_dir" ]]; then
		log_info "Cloning ${name} (fresh) from ${url}"
		safe_execute git clone "$url" "$target_dir"
	fi

	# Ensure origin URL is correct (do not delete the tree if URL changed)
	local current_url
	current_url=$(git -C "$target_dir" remote get-url origin 2>/dev/null || echo "")
	if [[ -n "$current_url" && "$current_url" != "$url" ]]; then
		log_info "Updating origin URL: ${current_url} -> ${url}"
		safe_execute git -C "$target_dir" remote set-url origin "$url"
	elif [[ -z "$current_url" ]]; then
		log_info "Setting origin URL to ${url}"
		safe_execute git -C "$target_dir" remote add origin "$url" || true
	fi

	# Fetch updates and tags
	safe_execute git -C "$target_dir" fetch --tags --prune origin

	# Resolve desired commit for the requested version/branch/tag
	local desired_rev=""
	if git -C "$target_dir" rev-parse --verify "${version}^{commit}" >/dev/null 2>&1; then
		desired_rev=$(git -C "$target_dir" rev-parse --verify "${version}^{commit}")
	elif git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}" >/dev/null 2>&1; then
		desired_rev=$(git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}")
	else
		log_warn "Version '${version}' not directly resolvable; fetching explicitly"
		if git -C "$target_dir" fetch origin "${version}" --depth 1; then
			desired_rev=$(git -C "$target_dir" rev-parse --verify FETCH_HEAD)
		fi
	fi

	# Fallback: shallow clone at the requested ref if we still can't resolve
	if [[ -z "$desired_rev" ]]; then
		log_warn "Could not resolve revision for '${version}'. Performing fresh shallow clone at requested ref."
		safe_execute rm -rf "${target_dir}.tmp"
		if safe_execute git clone --depth 1 --branch "$version" "$url" "${target_dir}.tmp"; then
			safe_execute rm -rf "$target_dir"
			safe_execute mv "${target_dir}.tmp" "$target_dir"
			desired_rev=$(git -C "$target_dir" rev-parse HEAD)
		else
			log_error "Failed to clone ${url} at '${version}'"
			return 1
		fi
	fi

	local current_rev
	current_rev=$(git -C "$target_dir" rev-parse HEAD 2>/dev/null || echo "")
	log_info "Current commit: ${current_rev:-<none>}"
	log_info "Desired  commit: ${desired_rev}"

	if [[ -n "$current_rev" && "$current_rev" == "$desired_rev" ]]; then
		log_info "Repository already at requested version; reusing working tree"
	else
		log_info "Checking out requested version"
		# Prefer named refs when available; otherwise detach to exact commit
		if git -C "$target_dir" show-ref --verify --quiet "refs/heads/${version}"; then
			safe_execute git -C "$target_dir" checkout -f "${version}"
		elif git -C "$target_dir" show-ref --verify --quiet "refs/remotes/origin/${version}"; then
			safe_execute git -C "$target_dir" checkout -f -B "${version}" "origin/${version}"
		elif git -C "$target_dir" show-ref --verify --quiet "refs/tags/${version}"; then
			safe_execute git -C "$target_dir" checkout -f "tags/${version}"
		else
			safe_execute git -C "$target_dir" checkout -f --detach "${desired_rev}"
		fi
		# Initialize submodules if present (non-fatal)
		safe_execute git -C "$target_dir" submodule update --init --recursive || true
	fi

	log_info "Git component ready: ${name} @ $(git -C "$target_dir" rev-parse --short HEAD)"
}

# Download release binary/archive
function components_download_release() {
	local name="$1"
	local url="$2"
	local version="$3"
	local components_dir="$4"
	local extra="$5"

	section_header "Downloading Release Component: ${name}"

	local target_dir="${components_dir}/${name}"
	local filename=$(basename "$url")

	log_info "Release URL: ${url}"
	log_info "Version: ${version}"
	log_info "Target directory: ${target_dir}"

	safe_mkdir "$target_dir"

	# Download release
	log_info "Downloading release: ${filename}"
	safe_execute wget --progress=dot:giga -O "${target_dir}/${filename}" "$url"

	# Verify download
	if [[ ! -f "${target_dir}/${filename}" ]]; then
		log_error "Failed to download release: ${filename}"
		return 1
	fi

	local file_size=$(get_file_size "${target_dir}/${filename}")
	log_info "Downloaded file size: ${file_size}"

	# Handle extra options (like rename)
	if [[ -n "$extra" ]]; then
		components_process_extra_options "$target_dir" "$filename" "$extra"
	fi

	log_info "Release component download complete: ${name}"
}

# Process extra options for components
function components_process_extra_options() {
	local target_dir="$1"
	local filename="$2"
	local extra="$3"

	log_info "Processing extra options: ${extra}"

	# Handle rename option
	if [[ "$extra" =~ rename=(.+) ]]; then
		local new_name="${BASH_REMATCH[1]}"
		log_info "Renaming ${filename} to ${new_name}"
		safe_execute mv "${target_dir}/${filename}" "${target_dir}/${new_name}"
	fi

	# Handle extract option for archives
	if [[ "$extra" =~ extract ]]; then
		log_info "Extracting archive: ${filename}"
		safe_execute cd "$target_dir"
		case "$filename" in
		*.tar.gz | *.tgz)
			safe_execute tar -xzf "$filename"
			;;
		*.tar.bz2 | *.tbz2)
			safe_execute tar -xjf "$filename"
			;;
		*.tar.xz | *.txz)
			safe_execute tar -xJf "$filename"
			;;
		*.zip)
			safe_execute unzip "$filename"
			;;
		*)
			log_warn "Unknown archive format: ${filename}"
			;;
		esac
	fi
}

# Build component using specified build function
function components_build_component() {
	local name="$1"
	local build_func="$2"
	local components_dir="$3"

	section_header "Building Component: ${name}"

	local component_dir="${components_dir}/${name}"

	if [[ ! -d "$component_dir" ]]; then
		log_error "Component directory not found: ${component_dir}"
		return 1
	fi

	# Change to component directory
	safe_execute cd "$component_dir"

	log_info "Build function: ${build_func}"
	log_info "Working directory: $(pwd)"

	# Check if build function exists
	if ! declare -f "$build_func" >/dev/null; then
		log_error "Build function not found: ${build_func}"
		return 1
	fi

	# Call the specific build function
	log_info "Executing build function: ${build_func}"
	"$build_func" "$name" "$component_dir"

	log_info "Component build complete: ${name}"
}

# Setup Rust environment for musl builds
function components_setup_rust_env() {
	section_header "Setting Up Rust Environment"

	# Source cargo environment if available
	if [[ -f /root/.cargo/env ]]; then
		log_info "Sourcing cargo environment from /root/.cargo/env"
		source /root/.cargo/env
	fi

	# Check if we have rustup (should be available now)
	if command_exists "rustup"; then
		log_info "Using rustup for Rust toolchain management"

		# Ensure musl target is installed
		if ! rustup target list --installed | grep -q "$RUST_TARGET"; then
			log_info "Installing Rust target: ${RUST_TARGET}"
			safe_execute rustup target add "$RUST_TARGET"
		else
			log_info "Rust target already installed: ${RUST_TARGET}"
		fi

		# Set environment variables for rustup (clean and simple)
		export RUSTFLAGS="-C target-feature=+crt-static"
	else
		log_error "rustup not found after setup"
		return 1
	fi

	log_info "Rust environment configured for musl builds"
	log_info "RUST_TARGET: ${RUST_TARGET}"
	log_info "RUSTFLAGS: ${RUSTFLAGS}"
	log_info "CC: ${CC:-system-default}"
}

# Build function for zinit (standard Rust build)
function build_zinit() {
	local name="$1"
	local component_dir="$2"

	section_header "Building zinit with musl target"

	components_setup_rust_env

	log_info "Building zinit from: ${component_dir}"

	# Ensure we're in the correct directory
	if [[ ! -d "$component_dir" ]]; then
		log_error "Component directory not found: ${component_dir}"
		return 1
	fi

	# Don't use safe_execute for cd - it runs in subshell
	log_info "Executing: cd $component_dir"
	cd "$component_dir" || {
		log_error "Failed to change to directory: $component_dir"
		return 1
	}

	local current_dir=$(pwd)
	log_info "Current directory: ${current_dir}"

	# Verify Cargo.toml exists
	if [[ ! -f "Cargo.toml" ]]; then
		log_error "Cargo.toml not found in: ${current_dir}"
		return 1
	fi

	# Build with musl target (rustup properly configured)
	safe_execute cargo build --release --target "$RUST_TARGET"

	# Find and install binary
	local binary_path="target/${RUST_TARGET}/release/zinit"
	if [[ ! -f "$binary_path" ]]; then
		log_error "zinit binary not found at: ${binary_path}"
		return 1
	fi

	local binary_size=$(get_file_size "$binary_path")
	log_info "Built zinit binary (${binary_size}) at: ${binary_path}"
}

# Build function for rfs (standard Rust build)
function build_rfs() {
	local name="$1"
	local component_dir="$2"

	section_header "Building rfs with musl target"

	components_setup_rust_env

	log_info "Building rfs from: ${component_dir}"

	# Ensure we're in the correct directory
	if [[ ! -d "$component_dir" ]]; then
		log_error "Component directory not found: ${component_dir}"
		return 1
	fi

	# Don't use safe_execute for cd - it runs in subshell
	log_info "Executing: cd $component_dir"
	cd "$component_dir" || {
		log_error "Failed to change to directory: $component_dir"
		return 1
	}

	local current_dir=$(pwd)
	log_info "Current directory: ${current_dir}"

	# Verify Cargo.toml exists
	if [[ ! -f "Cargo.toml" ]]; then
		log_error "Cargo.toml not found in: ${current_dir}"
		return 1
	fi
	# remove  rust-toolchain.toml, as not needed with latest release
	# Build with musl target
	safe_execute cargo build --release --target "$RUST_TARGET" --features build-binary

	# Find and install binary
	local binary_path="target/${RUST_TARGET}/release/rfs"
	if [[ ! -f "$binary_path" ]]; then
		log_error "rfs binary not found at: ${binary_path}"
		return 1
	fi

	local binary_size=$(get_file_size "$binary_path")
	log_info "Built rfs binary (${binary_size}) at: ${binary_path}"
}

# Build function for zosstorage (standard Rust build)
function build_zosstorage() {
	local name="$1"
	local component_dir="$2"

	section_header "Building zosstorage with musl target"

	components_setup_rust_env

	log_info "Building zosstorage from: ${component_dir}"

	if [[ ! -d "$component_dir" ]]; then
		log_error "Component directory not found: ${component_dir}"
		return 1
	fi

	log_info "Executing: cd $component_dir"
	cd "$component_dir" || {
		log_error "Failed to change to directory: $component_dir"
		return 1
	}

	local current_dir
	current_dir=$(pwd)
	log_info "Current directory: ${current_dir}"

	if [[ ! -f "Cargo.toml" ]]; then
		log_error "Cargo.toml not found in: ${current_dir}"
		return 1
	fi

	safe_execute cargo build --release --target "$RUST_TARGET"

	local binary_path="target/${RUST_TARGET}/release/zosstorage"
	if [[ ! -f "$binary_path" ]]; then
		log_error "zosstorage binary not found at: ${binary_path}"
		return 1
	fi

	local binary_size
	binary_size=$(get_file_size "$binary_path")
	log_info "Built zosstorage binary (${binary_size}) at: ${binary_path}"
}

# Build function for youki (standard Rust build)
function build_youki() {
	local name="$1"
	local component_dir="$2"

	section_header "Building youki with musl target"

	components_setup_rust_env

	log_info "Building youki from: ${component_dir}"

	if [[ ! -d "$component_dir" ]]; then
		log_error "Component directory not found: ${component_dir}"
		return 1
	fi

	log_info "Executing: cd $component_dir"
	cd "$component_dir" || {
		log_error "Failed to change to directory: $component_dir"
		return 1
	}

	local current_dir
	current_dir=$(pwd)
	log_info "Current directory: ${current_dir}"

	if [[ ! -f "Cargo.toml" ]]; then
		log_error "Cargo.toml not found in: ${current_dir}"
		return 1
	fi

	safe_execute cargo build --release --target "$RUST_TARGET"

	local binary_path="target/${RUST_TARGET}/release/youki"
	if [[ ! -f "$binary_path" ]]; then
		log_error "youki binary not found at: ${binary_path}"
		return 1
	fi

	local binary_size
	binary_size=$(get_file_size "$binary_path")
	log_info "Built youki binary (${binary_size}) at: ${binary_path}"
}
# Build function for mycelium (special subdirectory build)
function build_mycelium() {
	local name="$1"
	local component_dir="$2"

	section_header "Building mycelium with musl target (special directory)"

	components_setup_rust_env

	log_info "Building mycelium from: ${component_dir}"

	# Change to myceliumd subdirectory (special requirement)
	local myceliumd_dir="${component_dir}/myceliumd"
	if [[ ! -d "$myceliumd_dir" ]]; then
		log_error "myceliumd directory not found at: ${myceliumd_dir}"
		return 1
	fi

	# Don't use safe_execute for cd - it runs in subshell
	log_info "Executing: cd $myceliumd_dir"
	cd "$myceliumd_dir" || {
		log_error "Failed to change to myceliumd directory: $myceliumd_dir"
		return 1
	}
	log_info "Building in myceliumd subdirectory: $(pwd)"

	# Build with musl target
	safe_execute cargo build --release --target "$RUST_TARGET"

	# Find and install binary (from target/x86.../release)
	local binary_path="target/${RUST_TARGET}/release/mycelium"
	if [[ ! -f "$binary_path" ]]; then
		log_error "mycelium binary not found at: ${binary_path}"
		return 1
	fi

	local binary_size=$(get_file_size "$binary_path")
	log_info "Built mycelium binary (${binary_size}) at: ${binary_path}"
}

# Install function for rfs (pre-built binary)
function install_rfs() {
	local name="$1"
	local component_dir="$2"

	section_header "Installing rfs binary"

	log_info "Installing rfs from: ${component_dir}"

	# Find the rfs binary
	local binary_path="${component_dir}/rfs"
	if [[ ! -f "$binary_path" ]]; then
		log_error "rfs binary not found at: ${binary_path}"
		return 1
	fi

	# Make executable
	safe_execute chmod +x "$binary_path"

	local binary_size=$(get_file_size "$binary_path")
	log_info "Prepared rfs binary (${binary_size}) at: ${binary_path}"
}

# Install function for corex (pre-built binary)
function install_corex() {
	local name="$1"
	local component_dir="$2"

	section_header "Installing corex binary"

	log_info "Installing corex from: ${component_dir}"

	# Find the corex binary (may have been renamed)
	local binary_path
	if [[ -f "${component_dir}/corex" ]]; then
		binary_path="${component_dir}/corex"
	elif [[ -f "${component_dir}/corex-2.1.4-amd64-linux-static" ]]; then
		binary_path="${component_dir}/corex-2.1.4-amd64-linux-static"
	else
		log_error "corex binary not found in: ${component_dir}"
		return 1
	fi

	# Make executable
	safe_execute chmod +x "$binary_path"

	local binary_size=$(get_file_size "$binary_path")
	log_info "Prepared corex binary (${binary_size}) at: ${binary_path}"
}

# Verify all components are built (not copied to initramfs yet)
function components_verify_installation() {
	local components_dir="${COMPONENTS_DIR:-${PROJECT_ROOT}/components}"

	section_header "Verifying Component Build"

	local ok_count=0
	local missing_count=0

	# zinit
	local zinit_bin="${components_dir}/zinit/target/x86_64-unknown-linux-musl/release/zinit"
	if [[ -x "$zinit_bin" ]]; then
		log_info "✓ zinit ($(get_file_size "$zinit_bin")) at: ${zinit_bin#${components_dir}/}"
		((ok_count++))
	else
		log_error "✗ zinit missing: ${zinit_bin#${components_dir}/}"
		((missing_count++))
	fi

	# rfs: accept both built and prebuilt locations
	local rfs_built="${components_dir}/rfs/target/x86_64-unknown-linux-musl/release/rfs"
	local rfs_release="${components_dir}/rfs/rfs"
	if [[ -x "$rfs_built" ]]; then
		log_info "✓ rfs (built) ($(get_file_size "$rfs_built")) at: ${rfs_built#${components_dir}/}"
		((ok_count++))
	elif [[ -x "$rfs_release" ]]; then
		log_info "✓ rfs (release) ($(get_file_size "$rfs_release")) at: ${rfs_release#${components_dir}/}"
		((ok_count++))
	else
		log_error "✗ rfs missing: checked rfs/target/.../rfs and rfs/rfs"
		((missing_count++))
	fi

	# mycelium
	local mycelium_bin="${components_dir}/mycelium/myceliumd/target/x86_64-unknown-linux-musl/release/mycelium"
	if [[ -x "$mycelium_bin" ]]; then
		log_info "✓ mycelium ($(get_file_size "$mycelium_bin")) at: ${mycelium_bin#${components_dir}/}"
		((ok_count++))
	else
		log_error "✗ mycelium missing: ${mycelium_bin#${components_dir}/}"
		((missing_count++))
	fi

	# zosstorage
	local zosstorage_bin="${components_dir}/zosstorage/target/x86_64-unknown-linux-musl/release/zosstorage"
	if [[ -x "$zosstorage_bin" ]]; then
		log_info "✓ zosstorage ($(get_file_size "$zosstorage_bin")) at: ${zosstorage_bin#${components_dir}/}"
		((ok_count++))
	else
		log_error "✗ zosstorage missing: ${zosstorage_bin#${components_dir}/}"
		((missing_count++))
	fi

	# corex
	local corex_bin="${components_dir}/corex/corex"
	if [[ -x "$corex_bin" ]]; then
		log_info "✓ corex ($(get_file_size "$corex_bin")) at: ${corex_bin#${components_dir}/}"
		((ok_count++))
	else
		log_error "✗ corex missing: ${corex_bin#${components_dir}/}"
		((missing_count++))
	fi

	if [[ $missing_count -eq 0 ]]; then
		log_info "All components built/installed successfully"
		return 0
	else
		log_error "${missing_count} components missing or failed to build"
		return 1
	fi
}

# Clean component build artifacts
function components_cleanup() {
	local components_dir="$1"
	local keep_sources="${2:-false}"

	section_header "Cleaning Component Build Artifacts"

	if [[ "$keep_sources" == "true" ]]; then
		log_info "Keeping source directories, cleaning build artifacts only"

		# Clean Rust build artifacts
		find "$components_dir" -name "target" -type d -exec rm -rf {} + 2>/dev/null || true
		find "$components_dir" -name "Cargo.lock" -type f -delete 2>/dev/null || true

	else
		log_info "Removing all component directories"
		safe_rmdir "$components_dir"
	fi

	log_info "Component cleanup complete"
}

# Export functions
export -f components_parse_sources_conf
export -f components_download_git components_download_release components_process_extra_options
export -f components_build_component components_setup_rust_env
export -f build_zinit build_rfs build_zosstorage build_mycelium install_corex
export -f components_verify_installation components_cleanup
# Export functions for install_rfs
export -f install_rfs