tfgrid/zosbuilder
11
0
Fork 1
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
d85339cfd97b2182d5871c32671cf8336315e052
zosbuilder/docs/GITHUB.md

411 lines
11 KiB
Markdown
Raw Blame History

# GitHub Components Integration
Documentation for fetching and integrating the four custom components from GitHub repositories.
## Required GitHub Components
| Component | Repository | Purpose | Install Path |
|-----------|------------|---------|--------------|
| zinit | `threefoldtech/zinit` | Init system (PID 1) | `/sbin/zinit` |
| core-x | `threefoldtech/core-x` | Container remote control | `/usr/bin/core-x` |
| seektime | `threefoldtech/seektime` | Disk type detection | `/usr/bin/seektime` |
| rfs | `threefoldtech/rfs` | Rust version of 0-fs | `/usr/bin/rfs` |
## GitHub Release Fetcher Script
### scripts/fetch-github.sh
```bash
#!/bin/sh
set -e
GITHUB_DIR="/build/github"
INITRAMFS_ROOT="/build/initramfs"
CACHE_TIMEOUT=3600 # 1 hour cache
echo "[+] Fetching GitHub components..."
# Create cache directory
mkdir -p "$GITHUB_DIR"
# Function to get latest release info
get_latest_release() {
local repo="$1"
local cache_file="$GITHUB_DIR/${repo//\//_}_release.json"
# Check cache first
if [ -f "$cache_file" ] && [ $(($(date +%s) - $(stat -c %Y "$cache_file"))) -lt $CACHE_TIMEOUT ]; then
cat "$cache_file"
return
fi
# Fetch latest release info
local api_url="https://api.github.com/repos/$repo/releases/latest"
echo " Fetching release info for $repo..." >&2
if command -v curl >/dev/null; then
curl -s "$api_url" | tee "$cache_file"
else
wget -qO- "$api_url" | tee "$cache_file"
fi
}
# Function to extract download URL from release JSON
get_download_url() {
local release_json="$1"
local pattern="$2"
echo "$release_json" | grep -o '"browser_download_url": *"[^"]*"' | \
grep "$pattern" | head -1 | cut -d '"' -f 4
}
# Function to download and install binary
install_github_binary() {
local repo="$1"
local binary_name="$2"
local install_path="$3"
local download_pattern="$4"
echo " Installing: $repo -> $install_path/$binary_name"
# Get release information
local release_json=$(get_latest_release "$repo")
if [ -z "$release_json" ]; then
echo " Error: Failed to get release info for $repo" >&2
return 1
fi
# Extract version for logging
local version=$(echo "$release_json" | grep '"tag_name"' | cut -d '"' -f 4)
echo " Version: $version"
# Get download URL
local download_url=$(get_download_url "$release_json" "$download_pattern")
if [ -z "$download_url" ]; then
echo " Error: No matching release found for pattern: $download_pattern" >&2
return 1
fi
echo " Downloading: $(basename "$download_url")"
local filename=$(basename "$download_url")
local download_path="$GITHUB_DIR/$filename"
# Download if not cached
if [ ! -f "$download_path" ]; then
if command -v curl >/dev/null; then
curl -L -o "$download_path" "$download_url"
else
wget -O "$download_path" "$download_url"
fi
else
echo " Using cached file"
fi
# Create install directory
mkdir -p "$INITRAMFS_ROOT$install_path"
# Extract and install based on file type
case "$filename" in
*.tar.gz|*.tgz)
echo " Extracting tar.gz..."
tar -xzf "$download_path" -C "$GITHUB_DIR"
# Find the binary in extracted files
local extracted_binary=$(find "$GITHUB_DIR" -name "$binary_name" -type f | head -1)
if [ -n "$extracted_binary" ]; then
cp "$extracted_binary" "$INITRAMFS_ROOT$install_path/$binary_name"
else
echo " Error: Binary '$binary_name' not found in archive" >&2
return 1
fi
;;
*.zip)
echo " Extracting zip..."
unzip -q "$download_path" -d "$GITHUB_DIR"
local extracted_binary=$(find "$GITHUB_DIR" -name "$binary_name" -type f | head -1)
if [ -n "$extracted_binary" ]; then
cp "$extracted_binary" "$INITRAMFS_ROOT$install_path/$binary_name"
else
echo " Error: Binary '$binary_name' not found in archive" >&2
return 1
fi
;;
*)
# Assume it's a direct binary
echo " Installing direct binary..."
cp "$download_path" "$INITRAMFS_ROOT$install_path/$binary_name"
;;
esac
# Make executable
chmod +x "$INITRAMFS_ROOT$install_path/$binary_name"
# Verify installation
if [ -x "$INITRAMFS_ROOT$install_path/$binary_name" ]; then
echo " Success: $install_path/$binary_name installed"
else
echo " Error: Failed to install $binary_name" >&2
return 1
fi
}
# Install each component
echo ""
# Zinit - Init system
install_github_binary "threefoldtech/zinit" "zinit" "/sbin" "linux.*amd64"
# Core-X - Container control
install_github_binary "threefoldtech/core-x" "core-x" "/usr/bin" "linux.*amd64"
# Seektime - Disk detection
install_github_binary "threefoldtech/seektime" "seektime" "/usr/bin" "linux.*amd64"
# RFS - Rust filesystem
install_github_binary "threefoldtech/rfs" "rfs" "/usr/bin" "linux.*amd64"
echo ""
echo "[+] GitHub components installed successfully"
# Optional: Show installed binaries info
echo ""
echo "[+] Installed components:"
for binary in "/sbin/zinit" "/usr/bin/core-x" "/usr/bin/seektime" "/usr/bin/rfs"; do
if [ -x "$INITRAMFS_ROOT$binary" ]; then
size=$(stat -c%s "$INITRAMFS_ROOT$binary" 2>/dev/null || echo "unknown")
echo " $binary (${size} bytes)"
else
echo " $binary (MISSING)"
fi
done
```
## Alternative: Pinned Versions
For production builds, you may want to pin specific versions:
### scripts/fetch-github-pinned.sh
```bash
#!/bin/sh
set -e
GITHUB_DIR="/build/github"
INITRAMFS_ROOT="/build/initramfs"
# Pinned versions for reproducible builds
ZINIT_VERSION="v0.2.11"
COREX_VERSION="v2.1.0"
SEEKTIME_VERSION="v0.1"
RFS_VERSION="v1.1.1"
echo "[+] Fetching pinned GitHub components..."
# Function to download specific version
download_pinned_release() {
local repo="$1"
local version="$2"
local binary_name="$3"
local install_path="$4"
local asset_pattern="$5"
echo " Installing: $repo@$version"
# Construct release URL
local release_url="https://api.github.com/repos/$repo/releases/tags/$version"
# Get release info
local release_json
if command -v curl >/dev/null; then
release_json=$(curl -s "$release_url")
else
release_json=$(wget -qO- "$release_url")
fi
# Extract download URL
local download_url=$(echo "$release_json" | grep -o '"browser_download_url": *"[^"]*"' | \
grep "$asset_pattern" | head -1 | cut -d '"' -f 4)
if [ -z "$download_url" ]; then
echo " Error: No asset matching '$asset_pattern' found" >&2
return 1
fi
# Download and install
local filename=$(basename "$download_url")
local download_path="$GITHUB_DIR/$filename"
echo " Downloading: $filename"
if command -v curl >/dev/null; then
curl -L -o "$download_path" "$download_url"
else
wget -O "$download_path" "$download_url"
fi
# Install based on file type
mkdir -p "$INITRAMFS_ROOT$install_path"
case "$filename" in
*.tar.gz|*.tgz)
tar -xzf "$download_path" -C "$GITHUB_DIR"
cp "$GITHUB_DIR/$binary_name" "$INITRAMFS_ROOT$install_path/$binary_name"
;;
*)
cp "$download_path" "$INITRAMFS_ROOT$install_path/$binary_name"
;;
esac
chmod +x "$INITRAMFS_ROOT$install_path/$binary_name"
echo " Success: $install_path/$binary_name"
}
# Install pinned versions
download_pinned_release "threefoldtech/zinit" "$ZINIT_VERSION" "zinit" "/sbin" "linux.*amd64"
download_pinned_release "threefoldtech/core-x" "$COREX_VERSION" "core-x" "/usr/bin" "linux.*amd64"
download_pinned_release "threefoldtech/seektime" "$SEEKTIME_VERSION" "seektime" "/usr/bin" "linux.*amd64"
download_pinned_release "threefoldtech/rfs" "$RFS_VERSION" "rfs" "/usr/bin" "linux.*amd64"
echo "[+] Pinned GitHub components installed"
```
## Build Integration
### Integration with main build script:
```bash
# In build-initramfs.sh
echo "[+] Installing Alpine packages..."
/build/scripts/install-packages.sh
echo "[+] Fetching GitHub components..."
if [ "$BUILDMODE" = "production" ]; then
# Use pinned versions for production
/build/scripts/fetch-github-pinned.sh
else
# Use latest versions for development
/build/scripts/fetch-github.sh
fi
echo "[+] Setting up initramfs structure..."
/build/scripts/setup-initramfs.sh
```
## Caching Strategy
### Docker Volume Caching:
```yaml
# In docker-compose.yml
volumes:
github-cache:
driver: local
driver_opts:
type: none
o: bind
device: ./cache/github
services:
builder:
volumes:
- github-cache:/build/github
```
### Local Caching:
```bash
# Cache structure
alpine-initramfs/cache/
├── github/
│ ├── zinit-v0.2.11-linux-amd64.tar.gz
│ ├── core-x-v2.1.0-linux-amd64.tar.gz
│ ├── seektime-v0.1-linux-amd64.tar.gz
│ ├── rfs-v1.1.1-linux-amd64.tar.gz
│ └── *.json (release info cache)
```
## Error Handling
### Robust Download with Retries:
```bash
# Enhanced download function with retries
download_with_retry() {
local url="$1"
local output="$2"
local max_attempts=3
local attempt=1
while [ $attempt -le $max_attempts ]; do
echo " Attempt $attempt/$max_attempts..."
if command -v curl >/dev/null; then
if curl -L --fail -o "$output" "$url"; then
return 0
fi
else
if wget -O "$output" "$url"; then
return 0
fi
fi
attempt=$((attempt + 1))
if [ $attempt -le $max_attempts ]; then
echo " Retrying in 5 seconds..."
sleep 5
fi
done
echo " Error: Failed to download after $max_attempts attempts" >&2
return 1
}
```
## Verification
### Component Verification:
```bash
# Verify all components are installed and working
verify_components() {
local errors=0
echo "[+] Verifying GitHub components..."
# Check zinit
if [ -x "$INITRAMFS_ROOT/sbin/zinit" ]; then
echo " ✓ zinit installed"
else
echo " ✗ zinit missing"
errors=$((errors + 1))
fi
# Check core-x
if [ -x "$INITRAMFS_ROOT/usr/bin/core-x" ]; then
echo " ✓ core-x installed"
else
echo " ✗ core-x missing"
errors=$((errors + 1))
fi
# Check seektime
if [ -x "$INITRAMFS_ROOT/usr/bin/seektime" ]; then
echo " ✓ seektime installed"
else
echo " ✗ seektime missing"
errors=$((errors + 1))
fi
# Check rfs
if [ -x "$INITRAMFS_ROOT/usr/bin/rfs" ]; then
echo " ✓ rfs installed"
else
echo " ✗ rfs missing"
errors=$((errors + 1))
fi
if [ $errors -eq 0 ]; then
echo " All components verified successfully"
return 0
else
echo " $errors components missing"
return 1
fi
}
```
This system ensures reliable fetching and integration of the four custom GitHub components while maintaining caching and error resilience.
Reference in New Issue View Git Blame Copy Permalink