tfgrid/zosbuilder
11
0
Fork 1
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
72c484232499d091349ea4809674eb903565f149
zosbuilder/docs/GITHUB.md

11 KiB
Raw Blame History

GitHub Components Integration

Documentation for fetching and integrating the four custom components from GitHub repositories.

Required GitHub Components

Component Repository Purpose Install Path
zinit threefoldtech/zinit Init system (PID 1) /sbin/zinit
core-x threefoldtech/core-x Container remote control /usr/bin/core-x
seektime threefoldtech/seektime Disk type detection /usr/bin/seektime
rfs threefoldtech/rfs Rust version of 0-fs /usr/bin/rfs

GitHub Release Fetcher Script

scripts/fetch-github.sh

#!/bin/sh
set -e

GITHUB_DIR="/build/github"
INITRAMFS_ROOT="/build/initramfs"
CACHE_TIMEOUT=3600  # 1 hour cache

echo "[+] Fetching GitHub components..."

# Create cache directory
mkdir -p "$GITHUB_DIR"

# Function to get latest release info
get_latest_release() {
    local repo="$1"
    local cache_file="$GITHUB_DIR/${repo//\//_}_release.json"
    
    # Check cache first
    if [ -f "$cache_file" ] && [ $(($(date +%s) - $(stat -c %Y "$cache_file"))) -lt $CACHE_TIMEOUT ]; then
        cat "$cache_file"
        return
    fi
    
    # Fetch latest release info
    local api_url="https://api.github.com/repos/$repo/releases/latest"
    echo "    Fetching release info for $repo..." >&2
    
    if command -v curl >/dev/null; then
        curl -s "$api_url" | tee "$cache_file"
    else
        wget -qO- "$api_url" | tee "$cache_file"
    fi
}

# Function to extract download URL from release JSON
get_download_url() {
    local release_json="$1"
    local pattern="$2"
    
    echo "$release_json" | grep -o '"browser_download_url": *"[^"]*"' | \
        grep "$pattern" | head -1 | cut -d '"' -f 4
}

# Function to download and install binary
install_github_binary() {
    local repo="$1"
    local binary_name="$2"
    local install_path="$3"
    local download_pattern="$4"
    
    echo "  Installing: $repo -> $install_path/$binary_name"
    
    # Get release information
    local release_json=$(get_latest_release "$repo")
    if [ -z "$release_json" ]; then
        echo "    Error: Failed to get release info for $repo" >&2
        return 1
    fi
    
    # Extract version for logging
    local version=$(echo "$release_json" | grep '"tag_name"' | cut -d '"' -f 4)
    echo "    Version: $version"
    
    # Get download URL
    local download_url=$(get_download_url "$release_json" "$download_pattern")
    if [ -z "$download_url" ]; then
        echo "    Error: No matching release found for pattern: $download_pattern" >&2
        return 1
    fi
    
    echo "    Downloading: $(basename "$download_url")"
    local filename=$(basename "$download_url")
    local download_path="$GITHUB_DIR/$filename"
    
    # Download if not cached
    if [ ! -f "$download_path" ]; then
        if command -v curl >/dev/null; then
            curl -L -o "$download_path" "$download_url"
        else
            wget -O "$download_path" "$download_url"
        fi
    else
        echo "    Using cached file"
    fi
    
    # Create install directory
    mkdir -p "$INITRAMFS_ROOT$install_path"
    
    # Extract and install based on file type
    case "$filename" in
        *.tar.gz|*.tgz)
            echo "    Extracting tar.gz..."
            tar -xzf "$download_path" -C "$GITHUB_DIR"
            # Find the binary in extracted files
            local extracted_binary=$(find "$GITHUB_DIR" -name "$binary_name" -type f | head -1)
            if [ -n "$extracted_binary" ]; then
                cp "$extracted_binary" "$INITRAMFS_ROOT$install_path/$binary_name"
            else
                echo "    Error: Binary '$binary_name' not found in archive" >&2
                return 1
            fi
            ;;
        *.zip)
            echo "    Extracting zip..."
            unzip -q "$download_path" -d "$GITHUB_DIR"
            local extracted_binary=$(find "$GITHUB_DIR" -name "$binary_name" -type f | head -1)
            if [ -n "$extracted_binary" ]; then
                cp "$extracted_binary" "$INITRAMFS_ROOT$install_path/$binary_name"
            else
                echo "    Error: Binary '$binary_name' not found in archive" >&2
                return 1
            fi
            ;;
        *)
            # Assume it's a direct binary
            echo "    Installing direct binary..."
            cp "$download_path" "$INITRAMFS_ROOT$install_path/$binary_name"
            ;;
    esac
    
    # Make executable
    chmod +x "$INITRAMFS_ROOT$install_path/$binary_name"
    
    # Verify installation
    if [ -x "$INITRAMFS_ROOT$install_path/$binary_name" ]; then
        echo "    Success: $install_path/$binary_name installed"
    else
        echo "    Error: Failed to install $binary_name" >&2
        return 1
    fi
}

# Install each component
echo ""

# Zinit - Init system
install_github_binary "threefoldtech/zinit" "zinit" "/sbin" "linux.*amd64"

# Core-X - Container control
install_github_binary "threefoldtech/core-x" "core-x" "/usr/bin" "linux.*amd64"

# Seektime - Disk detection
install_github_binary "threefoldtech/seektime" "seektime" "/usr/bin" "linux.*amd64"

# RFS - Rust filesystem
install_github_binary "threefoldtech/rfs" "rfs" "/usr/bin" "linux.*amd64"

echo ""
echo "[+] GitHub components installed successfully"

# Optional: Show installed binaries info
echo ""
echo "[+] Installed components:"
for binary in "/sbin/zinit" "/usr/bin/core-x" "/usr/bin/seektime" "/usr/bin/rfs"; do
    if [ -x "$INITRAMFS_ROOT$binary" ]; then
        size=$(stat -c%s "$INITRAMFS_ROOT$binary" 2>/dev/null || echo "unknown")
        echo "  $binary (${size} bytes)"
    else
        echo "  $binary (MISSING)"
    fi
done

Alternative: Pinned Versions

For production builds, you may want to pin specific versions:

scripts/fetch-github-pinned.sh

#!/bin/sh
set -e

GITHUB_DIR="/build/github"
INITRAMFS_ROOT="/build/initramfs"

# Pinned versions for reproducible builds
ZINIT_VERSION="v0.2.11"
COREX_VERSION="v2.1.0"
SEEKTIME_VERSION="v0.1"
RFS_VERSION="v1.1.1"

echo "[+] Fetching pinned GitHub components..."

# Function to download specific version
download_pinned_release() {
    local repo="$1"
    local version="$2"
    local binary_name="$3"
    local install_path="$4"
    local asset_pattern="$5"
    
    echo "  Installing: $repo@$version"
    
    # Construct release URL
    local release_url="https://api.github.com/repos/$repo/releases/tags/$version"
    
    # Get release info
    local release_json
    if command -v curl >/dev/null; then
        release_json=$(curl -s "$release_url")
    else
        release_json=$(wget -qO- "$release_url")
    fi
    
    # Extract download URL
    local download_url=$(echo "$release_json" | grep -o '"browser_download_url": *"[^"]*"' | \
        grep "$asset_pattern" | head -1 | cut -d '"' -f 4)
    
    if [ -z "$download_url" ]; then
        echo "    Error: No asset matching '$asset_pattern' found" >&2
        return 1
    fi
    
    # Download and install
    local filename=$(basename "$download_url")
    local download_path="$GITHUB_DIR/$filename"
    
    echo "    Downloading: $filename"
    if command -v curl >/dev/null; then
        curl -L -o "$download_path" "$download_url"
    else
        wget -O "$download_path" "$download_url"
    fi
    
    # Install based on file type
    mkdir -p "$INITRAMFS_ROOT$install_path"
    
    case "$filename" in
        *.tar.gz|*.tgz)
            tar -xzf "$download_path" -C "$GITHUB_DIR"
            cp "$GITHUB_DIR/$binary_name" "$INITRAMFS_ROOT$install_path/$binary_name"
            ;;
        *)
            cp "$download_path" "$INITRAMFS_ROOT$install_path/$binary_name"
            ;;
    esac
    
    chmod +x "$INITRAMFS_ROOT$install_path/$binary_name"
    echo "    Success: $install_path/$binary_name"
}

# Install pinned versions
download_pinned_release "threefoldtech/zinit" "$ZINIT_VERSION" "zinit" "/sbin" "linux.*amd64"
download_pinned_release "threefoldtech/core-x" "$COREX_VERSION" "core-x" "/usr/bin" "linux.*amd64"
download_pinned_release "threefoldtech/seektime" "$SEEKTIME_VERSION" "seektime" "/usr/bin" "linux.*amd64"
download_pinned_release "threefoldtech/rfs" "$RFS_VERSION" "rfs" "/usr/bin" "linux.*amd64"

echo "[+] Pinned GitHub components installed"

Build Integration

Integration with main build script:

# In build-initramfs.sh
echo "[+] Installing Alpine packages..."
/build/scripts/install-packages.sh

echo "[+] Fetching GitHub components..."
if [ "$BUILDMODE" = "production" ]; then
    # Use pinned versions for production
    /build/scripts/fetch-github-pinned.sh
else
    # Use latest versions for development
    /build/scripts/fetch-github.sh
fi

echo "[+] Setting up initramfs structure..."
/build/scripts/setup-initramfs.sh

Caching Strategy

Docker Volume Caching:

# In docker-compose.yml
volumes:
  github-cache:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: ./cache/github

services:
  builder:
    volumes:
      - github-cache:/build/github

Local Caching:

# Cache structure
alpine-initramfs/cache/
├── github/
│   ├── zinit-v0.2.11-linux-amd64.tar.gz
│   ├── core-x-v2.1.0-linux-amd64.tar.gz
│   ├── seektime-v0.1-linux-amd64.tar.gz
│   ├── rfs-v1.1.1-linux-amd64.tar.gz
│   └── *.json (release info cache)

Error Handling

Robust Download with Retries:

# Enhanced download function with retries
download_with_retry() {
    local url="$1"
    local output="$2"
    local max_attempts=3
    local attempt=1
    
    while [ $attempt -le $max_attempts ]; do
        echo "    Attempt $attempt/$max_attempts..."
        
        if command -v curl >/dev/null; then
            if curl -L --fail -o "$output" "$url"; then
                return 0
            fi
        else
            if wget -O "$output" "$url"; then
                return 0
            fi
        fi
        
        attempt=$((attempt + 1))
        if [ $attempt -le $max_attempts ]; then
            echo "    Retrying in 5 seconds..."
            sleep 5
        fi
    done
    
    echo "    Error: Failed to download after $max_attempts attempts" >&2
    return 1
}

Verification

Component Verification:

# Verify all components are installed and working
verify_components() {
    local errors=0
    
    echo "[+] Verifying GitHub components..."
    
    # Check zinit
    if [ -x "$INITRAMFS_ROOT/sbin/zinit" ]; then
        echo "  ✓ zinit installed"
    else
        echo "  ✗ zinit missing"
        errors=$((errors + 1))
    fi
    
    # Check core-x
    if [ -x "$INITRAMFS_ROOT/usr/bin/core-x" ]; then
        echo "  ✓ core-x installed"
    else
        echo "  ✗ core-x missing"
        errors=$((errors + 1))
    fi
    
    # Check seektime
    if [ -x "$INITRAMFS_ROOT/usr/bin/seektime" ]; then
        echo "  ✓ seektime installed"
    else
        echo "  ✗ seektime missing"
        errors=$((errors + 1))
    fi
    
    # Check rfs
    if [ -x "$INITRAMFS_ROOT/usr/bin/rfs" ]; then
        echo "  ✓ rfs installed"
    else
        echo "  ✗ rfs missing"
        errors=$((errors + 1))
    fi
    
    if [ $errors -eq 0 ]; then
        echo "  All components verified successfully"
        return 0
    else
        echo "  $errors components missing"
        return 1
    fi
}

This system ensures reliable fetching and integration of the four custom GitHub components while maintaining caching and error resilience.