From 7eb27ff376677f52f8a5ab89b06c0714fd087fc8 Mon Sep 17 00:00:00 2001
From: Jan De Landtsheer <jan@delandtsheer.eu>
Date: Sat, 16 Aug 2025 21:04:07 +0200
Subject: [PATCH] Fix critical build pipeline and update documentation

CRITICAL FIXES:
- Fix build-initramfs.sh early exit preventing zinit installation
- Add binary detection/installation logic to setup-initramfs.sh
- Ensure compiled components are properly installed to initramfs

DOCUMENTATION:
- Update README.md to reflect actual implementation
- Document source compilation approach with git submodules
- Add development workflow and caching documentation
- Remove outdated GitHub releases references

This fixes the 'zinit not found' boot error by ensuring the build
pipeline completes all phases including initramfs assembly.
---
 README.md                  | 134 ++++++++++++++++++++++++++-----------
 scripts/build-initramfs.sh |   1 -
 scripts/setup-initramfs.sh |  44 ++++++++++++
 3 files changed, 140 insertions(+), 39 deletions(-)

diff --git a/README.md b/README.md
index 8c82487..5e226cc 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,16 @@
 # Alpine-based Zero-OS Initramfs
 
-Modern Alpine Linux based approach to building Zero-OS initramfs, replacing the complex build-from-source system with Alpine packages.
+Modern Alpine Linux based approach to building Zero-OS initramfs with source compilation for Zero-OS components and Alpine packages for system tools.
 
 ## Architecture Overview
 
 This system uses:
-- **Alpine Linux LTS** as base system (latest packages, no building from source)
-- **Alpine kernel** with embedded essential storage/ethernet modules  
-- **GitHub components**: zinit, core-x, seektime, rfs (only custom builds needed)
-- **Same init flow**: tmpfs copy approach for compatibility
-- **Alpine packages**: All standard tools (busybox, openssh, btrfs-progs, etc.)
+- **Alpine Linux 3.22** as base system with latest packages
+- **Git submodules** for Zero-OS components (zinit, mycelium, rfs, corex)
+- **Source compilation** for Zero-OS components (Rust/Go with musl static linking)
+- **Alpine packages** for system tools (busybox, openssh, btrfs-progs, etc.)
+- **Smart caching** with Docker layers and volumes
+- **Same init flow** as original Zero-OS
 
 ## Directory Structure
 
@@ -17,51 +18,108 @@ This system uses:
 alpine-initramfs/
 ├── build/                     # Build orchestration
 │   ├── Dockerfile.alpine      # Alpine build environment
-│   ├── docker-compose.yml     # Build orchestration
-│   └── build-initramfs.sh     # Main build script
+│   ├── Dockerfile.cached      # Multi-stage cached build
+│   └── docker-compose.yml     # Build orchestration with caching
+├── components/                # Git submodules
+│   ├── zinit/                 # Init system (Rust)
+│   ├── mycelium/              # Networking layer (Rust)
+│   ├── rfs/                   # Filesystem (Rust)
+│   └── corex/                 # Container control (static binary)
 ├── configs/                   # Configuration files
-│   ├── packages.txt           # Alpine packages to install
-│   ├── kernel-modules.txt     # Essential modules to embed
-│   ├── init                   # Custom init script (Alpine → zinit)
-│   └── zinit/                 # Zinit service configs (from ../config/etc/zinit/)
-├── scripts/                   # Build helper scripts
-│   ├── fetch-github.sh        # Download zinit, core-x, seektime, rfs
-│   ├── install-packages.sh    # Install Alpine packages to initramfs
+│   ├── packages-alpine.txt    # Alpine packages to install
+│   ├── kernel-version         # Kernel version to build
+│   ├── kernel-config-generic  # Kernel configuration
+│   ├── init                   # Init script (Alpine → zinit)
+│   └── zinit/                 # Zinit service configurations
+├── scripts/                   # Build scripts
+│   ├── build-initramfs.sh     # Main build orchestrator
+│   ├── compile-components.sh  # Compile Zero-OS components from source
+│   ├── install-packages.sh    # Install Alpine packages
 │   ├── setup-initramfs.sh     # Create initramfs structure
-│   └── build-kernel.sh        # Build kernel with embedded initramfs
+│   ├── build-kernel.sh        # Build kernel with embedded initramfs
+│   ├── build-smart.sh         # Smart build with caching
+│   ├── setup-submodules.sh    # Initialize git submodules
+│   └── cleanup.sh             # Clean build artifacts
 ├── output/                    # Build artifacts
-│   ├── initramfs.cpio.xz      # Generated initramfs
 │   ├── vmlinuz.efi            # Final bootable kernel
-│   └── modules/               # Kernel modules
+│   ├── initramfs.cpio.xz      # Generated initramfs
+│   └── version.txt            # Build information
 └── docs/                      # Documentation
-    ├── PACKAGES.md            # Package mapping (old build → Alpine)
-    ├── MODULES.md             # Kernel modules reference
-    └── BUILD.md               # Build process guide
+    ├── BUILD.md               # Build process guide
+    ├── OVERVIEW.md            # Architecture overview
+    └── *.md                   # Additional documentation
 ```
 
 ## Build Flow
 
-1. **Alpine Environment**: Setup Alpine LTS build container
-2. **Package Installation**: Install all required Alpine packages
-3. **GitHub Components**: Fetch zinit, core-x, seektime, rfs from releases
-4. **Initramfs Assembly**: Create filesystem structure with same tmpfs approach
-5. **Kernel Build**: Compile Alpine kernel with embedded essential modules
-6. **Output**: Generate bootable vmlinuz.efi with embedded initramfs
+1. **Environment Setup**: Alpine Docker container with Rust, Go, and build tools
+2. **Component Compilation**: Build zinit, mycelium, rfs from source (musl static)
+3. **Package Installation**: Install Alpine packages to initramfs root
+4. **Initramfs Assembly**: Create filesystem structure with compiled binaries
+5. **Kernel Build**: Compile kernel with embedded initramfs
+6. **Output**: Generate bootable vmlinuz.efi
 
 ## Quick Start
 
+### Full Build
 ```bash
-cd alpine-initramfs/build
-docker compose build
-docker compose run --rm builder
-# Output: ../output/vmlinuz.efi
+# Build everything
+./build.sh
+
+# Or with Docker Compose
+cd build/
+docker-compose up
 ```
 
-## Key Advantages
+### Development Build
+```bash
+cd build/
+# Interactive development shell
+docker-compose run dev-shell
 
-- ✅ **No source compilation** - Use Alpine's maintained packages
-- ✅ **Latest versions** - Always current with Alpine package updates  
-- ✅ **Faster builds** - Minutes vs hours
-- ✅ **Smaller footprint** - Alpine's minimal design
-- ✅ **Same functionality** - Identical init flow and service management
-- ✅ **Better maintenance** - Package updates handled by Alpine team
\ No newline at end of file
+# Build specific component (e.g., mycelium)
+cd /build/components/mycelium/myceliumd
+cargo build --release --target x86_64-unknown-linux-musl
+```
+
+### Smart Build (with caching)
+```bash
+cd build/
+docker-compose run builder  # Uses build-smart.sh automatically
+```
+
+## Key Features
+
+- ✅ **Source compilation** - Latest Zero-OS components with static linking
+- ✅ **Smart caching** - Docker layer and volume caching for fast rebuilds
+- ✅ **Alpine packages** - System tools from Alpine repositories
+- ✅ **Multi-stage builds** - Efficient Docker builds with minimal layers
+- ✅ **Development mode** - Interactive shell for component development
+- ✅ **Same functionality** - 100% compatible with original Zero-OS
+- ✅ **Modern toolchain** - Rust 1.x, Go latest, Alpine 3.22
+
+## Development
+
+### Building Individual Components
+```bash
+cd build/
+docker-compose run dev-shell
+
+# Inside container:
+cd /build/components/mycelium/myceliumd && cargo build --release --target x86_64-unknown-linux-musl
+cd /build/components/zinit && cargo build --release --target x86_64-unknown-linux-musl
+cd /build/components/rfs && cargo build --release --target x86_64-unknown-linux-musl
+```
+
+### Cleanup
+```bash
+# Clean all build artifacts
+./scripts/cleanup.sh
+```
+
+### Cache Management
+Smart build uses Docker volumes for caching:
+- `build-cache`: Build state markers
+- `source-cache`: Downloaded sources
+- `kernel-cache`: Kernel builds
+- `target-cache`: Rust build artifacts
\ No newline at end of file
diff --git a/scripts/build-initramfs.sh b/scripts/build-initramfs.sh
index f6478d0..410f428 100755
--- a/scripts/build-initramfs.sh
+++ b/scripts/build-initramfs.sh
@@ -42,7 +42,6 @@ echo "[+] Compiling Zero-OS components..."
 if [ -f /build/scripts/build-smart.sh ]; then
     # Use smart build if available (with caching)
     /build/scripts/build-smart.sh
-    exit $?
 else
     # Fallback to direct component compilation
     /build/scripts/compile-components.sh
diff --git a/scripts/setup-initramfs.sh b/scripts/setup-initramfs.sh
index 1736df0..1e8d7f2 100755
--- a/scripts/setup-initramfs.sh
+++ b/scripts/setup-initramfs.sh
@@ -85,6 +85,50 @@ EOF
 
 chmod +x "$INITRAMFS_ROOT/init-debug"
 
+# Ensure compiled Zero-OS binaries are installed
+echo "  Checking Zero-OS binaries..."
+echo "[DEBUG] Checking for compiled binaries..."
+
+# Check if binaries exist and copy them if needed
+BINARIES_FOUND=true
+
+# Check zinit (critical for boot)
+if [ ! -f "$INITRAMFS_ROOT/sbin/zinit" ]; then
+    if [ -f "/build/output/zinit" ]; then
+        echo "    Installing zinit from output directory..."
+        mkdir -p "$INITRAMFS_ROOT/sbin"
+        cp "/build/output/zinit" "$INITRAMFS_ROOT/sbin/"
+        chmod +x "$INITRAMFS_ROOT/sbin/zinit"
+    else
+        echo "    ERROR: zinit binary not found! Build may have failed."
+        BINARIES_FOUND=false
+    fi
+else
+    echo "    ✓ zinit found at $INITRAMFS_ROOT/sbin/zinit"
+fi
+
+# Check other binaries
+for binary in "usr/bin/rfs" "usr/bin/mycelium" "usr/bin/corex"; do
+    if [ ! -f "$INITRAMFS_ROOT/$binary" ]; then
+        basename_binary=$(basename "$binary")
+        if [ -f "/build/output/$basename_binary" ]; then
+            echo "    Installing $basename_binary from output directory..."
+            mkdir -p "$INITRAMFS_ROOT/$(dirname $binary)"
+            cp "/build/output/$basename_binary" "$INITRAMFS_ROOT/$binary"
+            chmod +x "$INITRAMFS_ROOT/$binary"
+        else
+            echo "    Warning: $basename_binary not found in output directory"
+        fi
+    else
+        echo "    ✓ $basename_binary found"
+    fi
+done
+
+if [ "$BINARIES_FOUND" = "false" ]; then
+    echo "[DEBUG] ERROR: Critical binaries missing - initramfs will not boot properly"
+    echo "[DEBUG] Make sure to run compile-components.sh before setup-initramfs.sh"
+fi
+
 # Copy other system configurations
 echo "  Setting up system configuration..."