#!/bin/bash # Component download and build system for ThreeFold Zero OS # Source common functions LIB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${LIB_SCRIPT_DIR}/common.sh" # Component configuration RUST_TARGET="${RUST_TARGET:-x86_64-unknown-linux-musl}" CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}" # Parse and process all components from sources.conf function components_parse_sources_conf() { local sources_file="$1" local components_dir="$2" local install_dir="${INSTALL_DIR:-${PROJECT_ROOT}/initramfs}" section_header "Parsing Sources Configuration" if [[ ! -f "$sources_file" ]]; then log_error "Sources file not found: ${sources_file}" return 1 fi # Ensure components directory exists safe_mkdir "$components_dir" # Export install directory for build functions export INSTALL_DIR="$install_dir" log_info "Processing components from: ${sources_file}" log_info "Components directory: ${components_dir}" log_info "Install directory: ${install_dir}" local component_count=0 # Read entries from sources.conf (TYPE NAME URL VERSION BUILD_FUNCTION [EXTRA]) while IFS= read -r _raw || [[ -n "$_raw" ]]; do # Strip comments and trim whitespace local line="${_raw%%#*}" line="${line#"${line%%[![:space:]]*}"}" line="${line%"${line##*[![:space:]]}"}" [[ -z "$line" ]] && continue local type name url version build_func extra # shellcheck disable=SC2086 read -r type name url version build_func extra <<< "$line" if [[ -z "${type:-}" || -z "${name:-}" || -z "${url:-}" || -z "${version:-}" || -z "${build_func:-}" ]]; then log_warn "Skipping malformed entry: ${_raw}" continue fi component_count=$((component_count + 1)) log_info "Processing component ${component_count}: ${name} (${type})" case "$type" in git) components_download_git "$name" "$url" "$version" "$components_dir" ;; release) components_download_release "$name" "$url" "$version" "$components_dir" "$extra" ;; *) log_error "Unknown component type in sources.conf: ${type}" return 1 ;; esac components_build_component "$name" "$build_func" "$components_dir" done < "$sources_file" if [[ $component_count -eq 0 ]]; then log_warn "No components found in sources configuration" else log_info "Processed ${component_count} components successfully" fi } # Download Git repository (reuse tree; only reclone if invalid or version not reachable) function components_download_git() { local name="$1" local url="$2" local version="$3" local components_dir="$4" section_header "Downloading Git Component: ${name}" local target_dir="${components_dir}/${name}" log_info "Repository: ${url}" log_info "Version/Branch/Tag: ${version}" log_info "Target directory: ${target_dir}" # Ensure parent exists safe_mkdir "$components_dir" # Decide whether we can reuse the existing working tree local need_fresh_clone="0" if [[ -d "$target_dir/.git" ]]; then if ! git -C "$target_dir" rev-parse --is-inside-work-tree >/dev/null 2>&1; then log_warn "Existing ${name} directory is not a valid git repo; will reclone" need_fresh_clone="1" fi elif [[ -d "$target_dir" ]]; then log_warn "Existing ${name} directory without .git; will reclone" need_fresh_clone="1" fi if [[ "$need_fresh_clone" == "1" || ! -d "$target_dir" ]]; then log_info "Cloning ${name} (fresh) from ${url}" safe_execute git clone "$url" "$target_dir" fi # Ensure origin URL is correct (do not delete the tree if URL changed) local current_url current_url=$(git -C "$target_dir" remote get-url origin 2>/dev/null || echo "") if [[ -n "$current_url" && "$current_url" != "$url" ]]; then log_info "Updating origin URL: ${current_url} -> ${url}" safe_execute git -C "$target_dir" remote set-url origin "$url" elif [[ -z "$current_url" ]]; then log_info "Setting origin URL to ${url}" safe_execute git -C "$target_dir" remote add origin "$url" || true fi # Fetch updates and tags safe_execute git -C "$target_dir" fetch --tags --prune origin # Resolve desired commit for the requested version/branch/tag local desired_rev="" if git -C "$target_dir" rev-parse --verify "${version}^{commit}" >/dev/null 2>&1; then desired_rev=$(git -C "$target_dir" rev-parse --verify "${version}^{commit}") elif git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}" >/dev/null 2>&1; then desired_rev=$(git -C "$target_dir" rev-parse --verify "origin/${version}^{commit}") else log_warn "Version '${version}' not directly resolvable; fetching explicitly" if git -C "$target_dir" fetch origin "${version}" --depth 1; then desired_rev=$(git -C "$target_dir" rev-parse --verify FETCH_HEAD) fi fi # Fallback: shallow clone at the requested ref if we still can't resolve if [[ -z "$desired_rev" ]]; then log_warn "Could not resolve revision for '${version}'. Performing fresh shallow clone at requested ref." safe_execute rm -rf "${target_dir}.tmp" if safe_execute git clone --depth 1 --branch "$version" "$url" "${target_dir}.tmp"; then safe_execute rm -rf "$target_dir" safe_execute mv "${target_dir}.tmp" "$target_dir" desired_rev=$(git -C "$target_dir" rev-parse HEAD) else log_error "Failed to clone ${url} at '${version}'" return 1 fi fi local current_rev current_rev=$(git -C "$target_dir" rev-parse HEAD 2>/dev/null || echo "") log_info "Current commit: ${current_rev:-}" log_info "Desired commit: ${desired_rev}" if [[ -n "$current_rev" && "$current_rev" == "$desired_rev" ]]; then log_info "Repository already at requested version; reusing working tree" else log_info "Checking out requested version" # Prefer named refs when available; otherwise detach to exact commit if git -C "$target_dir" show-ref --verify --quiet "refs/heads/${version}"; then safe_execute git -C "$target_dir" checkout -f "${version}" elif git -C "$target_dir" show-ref --verify --quiet "refs/remotes/origin/${version}"; then safe_execute git -C "$target_dir" checkout -f -B "${version}" "origin/${version}" elif git -C "$target_dir" show-ref --verify --quiet "refs/tags/${version}"; then safe_execute git -C "$target_dir" checkout -f "tags/${version}" else safe_execute git -C "$target_dir" checkout -f --detach "${desired_rev}" fi # Initialize submodules if present (non-fatal) safe_execute git -C "$target_dir" submodule update --init --recursive || true fi log_info "Git component ready: ${name} @ $(git -C "$target_dir" rev-parse --short HEAD)" } # Download release binary/archive function components_download_release() { local name="$1" local url="$2" local version="$3" local components_dir="$4" local extra="$5" section_header "Downloading Release Component: ${name}" local target_dir="${components_dir}/${name}" local filename=$(basename "$url") log_info "Release URL: ${url}" log_info "Version: ${version}" log_info "Target directory: ${target_dir}" safe_mkdir "$target_dir" # Download release log_info "Downloading release: ${filename}" safe_execute wget --progress=dot:giga -O "${target_dir}/${filename}" "$url" # Verify download if [[ ! -f "${target_dir}/${filename}" ]]; then log_error "Failed to download release: ${filename}" return 1 fi local file_size=$(get_file_size "${target_dir}/${filename}") log_info "Downloaded file size: ${file_size}" # Handle extra options (like rename) if [[ -n "$extra" ]]; then components_process_extra_options "$target_dir" "$filename" "$extra" fi log_info "Release component download complete: ${name}" } # Process extra options for components function components_process_extra_options() { local target_dir="$1" local filename="$2" local extra="$3" log_info "Processing extra options: ${extra}" # Handle rename option if [[ "$extra" =~ rename=(.+) ]]; then local new_name="${BASH_REMATCH[1]}" log_info "Renaming ${filename} to ${new_name}" safe_execute mv "${target_dir}/${filename}" "${target_dir}/${new_name}" fi # Handle extract option for archives if [[ "$extra" =~ extract ]]; then log_info "Extracting archive: ${filename}" safe_execute cd "$target_dir" case "$filename" in *.tar.gz|*.tgz) safe_execute tar -xzf "$filename" ;; *.tar.bz2|*.tbz2) safe_execute tar -xjf "$filename" ;; *.tar.xz|*.txz) safe_execute tar -xJf "$filename" ;; *.zip) safe_execute unzip "$filename" ;; *) log_warn "Unknown archive format: ${filename}" ;; esac fi } # Build component using specified build function function components_build_component() { local name="$1" local build_func="$2" local components_dir="$3" section_header "Building Component: ${name}" local component_dir="${components_dir}/${name}" if [[ ! -d "$component_dir" ]]; then log_error "Component directory not found: ${component_dir}" return 1 fi # Change to component directory safe_execute cd "$component_dir" log_info "Build function: ${build_func}" log_info "Working directory: $(pwd)" # Check if build function exists if ! declare -f "$build_func" >/dev/null; then log_error "Build function not found: ${build_func}" return 1 fi # Call the specific build function log_info "Executing build function: ${build_func}" "$build_func" "$name" "$component_dir" log_info "Component build complete: ${name}" } # Setup Rust environment for musl builds function components_setup_rust_env() { section_header "Setting Up Rust Environment" # Source cargo environment if available if [[ -f /root/.cargo/env ]]; then log_info "Sourcing cargo environment from /root/.cargo/env" source /root/.cargo/env fi # Check if we have rustup (should be available now) if command_exists "rustup"; then log_info "Using rustup for Rust toolchain management" # Ensure musl target is installed if ! rustup target list --installed | grep -q "$RUST_TARGET"; then log_info "Installing Rust target: ${RUST_TARGET}" safe_execute rustup target add "$RUST_TARGET" else log_info "Rust target already installed: ${RUST_TARGET}" fi # Set environment variables for rustup (clean and simple) export RUSTFLAGS="-C target-feature=+crt-static" else log_error "rustup not found after setup" return 1 fi log_info "Rust environment configured for musl builds" log_info "RUST_TARGET: ${RUST_TARGET}" log_info "RUSTFLAGS: ${RUSTFLAGS}" log_info "CC: ${CC:-system-default}" } # Build function for zinit (standard Rust build) function build_zinit() { local name="$1" local component_dir="$2" section_header "Building zinit with musl target" components_setup_rust_env log_info "Building zinit from: ${component_dir}" # Ensure we're in the correct directory if [[ ! -d "$component_dir" ]]; then log_error "Component directory not found: ${component_dir}" return 1 fi # Don't use safe_execute for cd - it runs in subshell log_info "Executing: cd $component_dir" cd "$component_dir" || { log_error "Failed to change to directory: $component_dir" return 1 } local current_dir=$(pwd) log_info "Current directory: ${current_dir}" # Verify Cargo.toml exists if [[ ! -f "Cargo.toml" ]]; then log_error "Cargo.toml not found in: ${current_dir}" return 1 fi # Build with musl target (rustup properly configured) safe_execute cargo build --release --target "$RUST_TARGET" # Find and install binary local binary_path="target/${RUST_TARGET}/release/zinit" if [[ ! -f "$binary_path" ]]; then log_error "zinit binary not found at: ${binary_path}" return 1 fi local binary_size=$(get_file_size "$binary_path") log_info "Built zinit binary (${binary_size}) at: ${binary_path}" } # Build function for rfs (standard Rust build) function build_rfs() { local name="$1" local component_dir="$2" section_header "Building rfs with musl target" components_setup_rust_env log_info "Building rfs from: ${component_dir}" # Ensure we're in the correct directory if [[ ! -d "$component_dir" ]]; then log_error "Component directory not found: ${component_dir}" return 1 fi # Don't use safe_execute for cd - it runs in subshell log_info "Executing: cd $component_dir" cd "$component_dir" || { log_error "Failed to change to directory: $component_dir" return 1 } local current_dir=$(pwd) log_info "Current directory: ${current_dir}" # Verify Cargo.toml exists if [[ ! -f "Cargo.toml" ]]; then log_error "Cargo.toml not found in: ${current_dir}" return 1 fi # remove rust-toolchain.toml, as not needed with latest release # Build with musl target safe_execute cargo build --release --target "$RUST_TARGET" --features build-binary # Find and install binary local binary_path="target/${RUST_TARGET}/release/rfs" if [[ ! -f "$binary_path" ]]; then log_error "rfs binary not found at: ${binary_path}" return 1 fi local binary_size=$(get_file_size "$binary_path") log_info "Built rfs binary (${binary_size}) at: ${binary_path}" } # Build function for zosstorage (standard Rust build) function build_zosstorage() { local name="$1" local component_dir="$2" section_header "Building zosstorage with musl target" components_setup_rust_env log_info "Building zosstorage from: ${component_dir}" if [[ ! -d "$component_dir" ]]; then log_error "Component directory not found: ${component_dir}" return 1 fi log_info "Executing: cd $component_dir" cd "$component_dir" || { log_error "Failed to change to directory: $component_dir" return 1 } local current_dir current_dir=$(pwd) log_info "Current directory: ${current_dir}" if [[ ! -f "Cargo.toml" ]]; then log_error "Cargo.toml not found in: ${current_dir}" return 1 fi safe_execute cargo build --release --target "$RUST_TARGET" local binary_path="target/${RUST_TARGET}/release/zosstorage" if [[ ! -f "$binary_path" ]]; then log_error "zosstorage binary not found at: ${binary_path}" return 1 fi local binary_size binary_size=$(get_file_size "$binary_path") log_info "Built zosstorage binary (${binary_size}) at: ${binary_path}" } # Build function for mycelium (special subdirectory build) function build_mycelium() { local name="$1" local component_dir="$2" section_header "Building mycelium with musl target (special directory)" components_setup_rust_env log_info "Building mycelium from: ${component_dir}" # Change to myceliumd subdirectory (special requirement) local myceliumd_dir="${component_dir}/myceliumd" if [[ ! -d "$myceliumd_dir" ]]; then log_error "myceliumd directory not found at: ${myceliumd_dir}" return 1 fi # Don't use safe_execute for cd - it runs in subshell log_info "Executing: cd $myceliumd_dir" cd "$myceliumd_dir" || { log_error "Failed to change to myceliumd directory: $myceliumd_dir" return 1 } log_info "Building in myceliumd subdirectory: $(pwd)" # Build with musl target safe_execute cargo build --release --target "$RUST_TARGET" # Find and install binary (from target/x86.../release) local binary_path="target/${RUST_TARGET}/release/mycelium" if [[ ! -f "$binary_path" ]]; then log_error "mycelium binary not found at: ${binary_path}" return 1 fi local binary_size=$(get_file_size "$binary_path") log_info "Built mycelium binary (${binary_size}) at: ${binary_path}" } # Install function for rfs (pre-built binary) function install_rfs() { local name="$1" local component_dir="$2" section_header "Installing rfs binary" log_info "Installing rfs from: ${component_dir}" # Find the rfs binary local binary_path="${component_dir}/rfs" if [[ ! -f "$binary_path" ]]; then log_error "rfs binary not found at: ${binary_path}" return 1 fi # Make executable safe_execute chmod +x "$binary_path" local binary_size=$(get_file_size "$binary_path") log_info "Prepared rfs binary (${binary_size}) at: ${binary_path}" } # Install function for corex (pre-built binary) function install_corex() { local name="$1" local component_dir="$2" section_header "Installing corex binary" log_info "Installing corex from: ${component_dir}" # Find the corex binary (may have been renamed) local binary_path if [[ -f "${component_dir}/corex" ]]; then binary_path="${component_dir}/corex" elif [[ -f "${component_dir}/corex-2.1.4-amd64-linux-static" ]]; then binary_path="${component_dir}/corex-2.1.4-amd64-linux-static" else log_error "corex binary not found in: ${component_dir}" return 1 fi # Make executable safe_execute chmod +x "$binary_path" local binary_size=$(get_file_size "$binary_path") log_info "Prepared corex binary (${binary_size}) at: ${binary_path}" } # Verify all components are built (not copied to initramfs yet) function components_verify_installation() { local components_dir="${COMPONENTS_DIR:-${PROJECT_ROOT}/components}" section_header "Verifying Component Build" local ok_count=0 local missing_count=0 # zinit local zinit_bin="${components_dir}/zinit/target/x86_64-unknown-linux-musl/release/zinit" if [[ -x "$zinit_bin" ]]; then log_info "✓ zinit ($(get_file_size "$zinit_bin")) at: ${zinit_bin#${components_dir}/}" ((ok_count++)) else log_error "✗ zinit missing: ${zinit_bin#${components_dir}/}" ((missing_count++)) fi # rfs: accept both built and prebuilt locations local rfs_built="${components_dir}/rfs/target/x86_64-unknown-linux-musl/release/rfs" local rfs_release="${components_dir}/rfs/rfs" if [[ -x "$rfs_built" ]]; then log_info "✓ rfs (built) ($(get_file_size "$rfs_built")) at: ${rfs_built#${components_dir}/}" ((ok_count++)) elif [[ -x "$rfs_release" ]]; then log_info "✓ rfs (release) ($(get_file_size "$rfs_release")) at: ${rfs_release#${components_dir}/}" ((ok_count++)) else log_error "✗ rfs missing: checked rfs/target/.../rfs and rfs/rfs" ((missing_count++)) fi # mycelium local mycelium_bin="${components_dir}/mycelium/myceliumd/target/x86_64-unknown-linux-musl/release/mycelium" if [[ -x "$mycelium_bin" ]]; then log_info "✓ mycelium ($(get_file_size "$mycelium_bin")) at: ${mycelium_bin#${components_dir}/}" ((ok_count++)) else log_error "✗ mycelium missing: ${mycelium_bin#${components_dir}/}" ((missing_count++)) fi # zosstorage local zosstorage_bin="${components_dir}/zosstorage/target/x86_64-unknown-linux-musl/release/zosstorage" if [[ -x "$zosstorage_bin" ]]; then log_info "✓ zosstorage ($(get_file_size "$zosstorage_bin")) at: ${zosstorage_bin#${components_dir}/}" ((ok_count++)) else log_error "✗ zosstorage missing: ${zosstorage_bin#${components_dir}/}" ((missing_count++)) fi # corex local corex_bin="${components_dir}/corex/corex" if [[ -x "$corex_bin" ]]; then log_info "✓ corex ($(get_file_size "$corex_bin")) at: ${corex_bin#${components_dir}/}" ((ok_count++)) else log_error "✗ corex missing: ${corex_bin#${components_dir}/}" ((missing_count++)) fi if [[ $missing_count -eq 0 ]]; then log_info "All components built/installed successfully" return 0 else log_error "${missing_count} components missing or failed to build" return 1 fi } # Clean component build artifacts function components_cleanup() { local components_dir="$1" local keep_sources="${2:-false}" section_header "Cleaning Component Build Artifacts" if [[ "$keep_sources" == "true" ]]; then log_info "Keeping source directories, cleaning build artifacts only" # Clean Rust build artifacts find "$components_dir" -name "target" -type d -exec rm -rf {} + 2>/dev/null || true find "$components_dir" -name "Cargo.lock" -type f -delete 2>/dev/null || true else log_info "Removing all component directories" safe_rmdir "$components_dir" fi log_info "Component cleanup complete" } # Export functions export -f components_parse_sources_conf export -f components_download_git components_download_release components_process_extra_options export -f components_build_component components_setup_rust_env export -f build_zinit build_rfs build_zosstorage build_mycelium install_corex export -f components_verify_installation components_cleanup # Export functions for install_rfs export -f install_rfs