build/rfs: integrate RFS flists + runtime orchestration

• Add standalone RFS tooling: scripts/rfs/common.sh, pack-modules.sh, pack-firmware.sh, verify-flist.sh

• Patch flist route.url with read-only Garage S3 credentials; optional HTTPS store row; optional manifest upload via mcli

• Build integration: stage_rfs_flists in scripts/build.sh to pack and embed manifests under initramfs/etc/rfs

• Runtime: add zinit units rfs-modules (after: network), rfs-firmware (after: network) as daemons; add udev-rfs oneshot post-mount

• Keep early udev-trigger oneshot to coldplug NICs before RFS mounts

• Firmware flist reproducible naming: respect FIRMWARE_TAG from env or config/build.conf, default to latest

• Docs: update docs/rfs-flists.md with runtime ordering, reproducible tagging, verification steps

config/build.conf

@@ -34,6 +34,15 @@ DIST_DIR="dist"
 ALPINE_MIRROR="https://dl-cdn.alpinelinux.org/alpine"
 KERNEL_SOURCE_URL="https://cdn.kernel.org/pub/linux/kernel"
 
+# RFS flists (firmware manifest naming)
+# FIRMWARE_TAG controls firmware flist manifest naming for reproducible builds.
+# - If set, firmware manifest becomes: firmware-$FIRMWARE_TAG.fl
+# - If unset, the build embeds firmware-latest.fl, while standalone pack may default to date-based naming.
+# Examples:
+# FIRMWARE_TAG="20250908"
+# FIRMWARE_TAG="v1"
+#FIRMWARE_TAG="latest"
+
 # Feature flags
 ENABLE_STRIP="true"
 ENABLE_UPX="true"

config/zinit/rfs-firmware.yaml

@@ -0,0 +1,4 @@
+exec: sh /etc/zinit/init/firmware.sh
+restart: always
+after:
+  - network

config/zinit/rfs-modules.yaml

@@ -0,0 +1,4 @@
+exec: sh /etc/zinit/init/modules.sh
+restart: always
+after:
+  - network

config/zinit/udev-rfs.yaml

@@ -0,0 +1,5 @@
+exec: /bin/sh -c "udevadm control --reload; udevadm trigger --action=add --type=subsystems; udevadm trigger --action=add --type=devices; udevadm settle"
+oneshot: true
+after:
+  - rfs-modules
+  - rfs-firmware

docs/rfs-flists.md

@@ -110,3 +110,57 @@ Note on route URL vs HTTP endpoint
   - READ_ACCESS_KEY / READ_SECRET_KEY: read-only credentials
   - ROUTE_ENDPOINT (defaults to S3_ENDPOINT), ROUTE_PATH=/blobs, ROUTE_REGION=garage
 - Do not set ROUTE_PATH to S3_PREFIX. ROUTE_PATH is the gateway’s blob route (usually /blobs). S3_PREFIX is only for the pack-time store path.
+
+## Runtime units and ordering (zinit)
+
+This repo now includes runtime zinit units and init scripts to mount the RFS flists and perform dual udev coldplug sequences.
+
+- Early coldplug (before RFS mounts):
+  - [config/zinit/udev-trigger.yaml](config/zinit/udev-trigger.yaml) calls [config/zinit/init/udev.sh](config/zinit/init/udev.sh).
+  - Runs after depmod/udev daemons to initialize NICs and other devices using what is already in the initramfs.
+  - Purpose: bring up networking so RFS can reach Garage S3.
+
+- RFS mounts (daemons, after network):
+  - [config/zinit/rfs-modules.yaml](config/zinit/rfs-modules.yaml) runs [config/zinit/init/modules.sh](config/zinit/init/modules.sh) to mount modules-$(uname -r).fl onto /lib/modules/$(uname -r).
+  - [config/zinit/rfs-firmware.yaml](config/zinit/rfs-firmware.yaml) runs [config/zinit/init/firmware.sh](config/zinit/init/firmware.sh) to mount firmware-latest.fl onto /usr/lib/firmware.
+  - Both are defined as restart: always and include after: network to ensure the Garage S3 route is reachable.
+
+- Post-mount coldplug (after RFS mounts):
+  - [config/zinit/udev-rfs.yaml](config/zinit/udev-rfs.yaml) performs:
+    - udevadm control --reload
+    - udevadm trigger --action=add --type=subsystems
+    - udevadm trigger --action=add --type=devices
+    - udevadm settle
+  - This re-probes hardware so new modules/firmware from the overmounted flists are considered.
+
+- Embedded manifests in initramfs:
+  - The build embeds the flists under /etc/rfs:
+    - modules-KERNEL_FULL_VERSION.fl
+    - firmware-latest.fl
+  - Creation happens in [scripts/rfs/pack-modules.sh](scripts/rfs/pack-modules.sh) and [scripts/rfs/pack-firmware.sh](scripts/rfs/pack-firmware.sh), and embedding is orchestrated by [scripts/build.sh](scripts/build.sh).
+
+## Reproducible firmware tagging
+
+- The firmware flist name can be pinned via FIRMWARE_TAG in [config/build.conf](config/build.conf).
+  - If set: firmware-FIRMWARE_TAG.fl
+  - If unset: the build uses firmware-latest.fl for embedding (standalone pack may default to date-based).
+- The build logic picks the tag with this precedence:
+  1) Environment FIRMWARE_TAG
+  2) FIRMWARE_TAG from [config/build.conf](config/build.conf)
+  3) "latest"
+- Build integration implemented in [scripts/build.sh](scripts/build.sh).
+
+Example:
+- Set FIRMWARE_TAG in config: add FIRMWARE_TAG="20250908" in [config/build.conf](config/build.conf)
+- Or export at build time: export FIRMWARE_TAG="v1"
+
+## Verifying flists
+
+Use the helper to inspect a manifest, optionally listing entries and testing a local mount (root + proper FUSE policy required):
+
+- Inspect only:
+  - scripts/rfs/verify-flist.sh -m dist/flists/modules-6.12.44-Zero-OS.fl
+- Inspect + tree:
+  - scripts/rfs/verify-flist.sh -m dist/flists/firmware-latest.fl --tree
+- Inspect + mount test to a temp dir:
+  - sudo scripts/rfs/verify-flist.sh -m dist/flists/modules-6.12.44-Zero-OS.fl --mount

scripts/build.sh

@@ -287,6 +287,62 @@ function main_build_process() {
         initramfs_copy_resolved_modules "$INSTALL_DIR" "$FULL_KERNEL_VERSION"
     }
     
+    # Create RFS flists and embed them into initramfs prior to CPIO
+    function stage_rfs_flists() {
+        section_header "Creating RFS flists and embedding into initramfs"
+
+        # Ensure FULL_KERNEL_VERSION is available
+        if [[ -z "${FULL_KERNEL_VERSION:-}" ]]; then
+            FULL_KERNEL_VERSION=$(kernel_get_full_version "$KERNEL_VERSION" "$KERNEL_CONFIG")
+            export FULL_KERNEL_VERSION
+            log_info "Resolved FULL_KERNEL_VERSION: ${FULL_KERNEL_VERSION}"
+        fi
+
+        # Ensure rfs scripts are executable (avoid subshell to preserve quoting)
+        safe_execute chmod +x ./scripts/rfs/*.sh
+
+        # Build modules flist (writes to dist/flists/modules-${FULL_KERNEL_VERSION}.fl)
+        safe_execute ./scripts/rfs/pack-modules.sh
+
+        # Build firmware flist with a reproducible tag:
+        # Priority: env FIRMWARE_TAG > config/build.conf: FIRMWARE_TAG > "latest"
+        local fw_tag
+        if [[ -n "${FIRMWARE_TAG:-}" ]]; then
+            fw_tag="${FIRMWARE_TAG}"
+        else
+            if [[ -f "${CONFIG_DIR}/build.conf" ]]; then
+                # shellcheck source=/dev/null
+                source "${CONFIG_DIR}/build.conf"
+            fi
+            fw_tag="${FIRMWARE_TAG:-latest}"
+        fi
+        log_info "Using firmware tag: ${fw_tag}"
+        safe_execute env FIRMWARE_TAG="${fw_tag}" ./scripts/rfs/pack-firmware.sh
+
+        # Embed flists inside initramfs at /etc/rfs for zinit init scripts
+        local etc_rfs_dir="${INSTALL_DIR}/etc/rfs"
+        safe_mkdir "${etc_rfs_dir}"
+
+        local modules_fl="dist/flists/modules-${FULL_KERNEL_VERSION}.fl"
+        if [[ -f "${modules_fl}" ]]; then
+            safe_execute cp "${modules_fl}" "${etc_rfs_dir}/"
+            log_info "Embedded modules flist: ${modules_fl} -> ${etc_rfs_dir}/"
+        else
+            log_warn "Modules flist not found: ${modules_fl}"
+        fi
+
+        local firmware_fl="dist/flists/firmware-${fw_tag}.fl"
+        if [[ -f "${firmware_fl}" ]]; then
+            # Provide canonical name firmware-latest.fl expected by firmware.sh
+            safe_execute cp "${firmware_fl}" "${etc_rfs_dir}/firmware-latest.fl"
+            log_info "Embedded firmware flist: ${firmware_fl} -> ${etc_rfs_dir}/firmware-latest.fl"
+        else
+            log_warn "Firmware flist not found: ${firmware_fl}"
+        fi
+
+        log_info "RFS flists embedded into initramfs"
+    }
+
     function stage_cleanup() {
         alpine_aggressive_cleanup "$INSTALL_DIR"
     }
@@ -336,6 +392,7 @@ function main_build_process() {
     stage_run "modules_setup" stage_modules_setup
     stage_run "modules_copy" stage_modules_copy
     stage_run "cleanup" stage_cleanup
+    stage_run "rfs_flists" stage_rfs_flists
     stage_run "validation" stage_validation
     stage_run "initramfs_create" stage_initramfs_create
     stage_run "initramfs_test" stage_initramfs_test

scripts/rfs/verify-flist.sh

@@ -1,12 +1,6 @@
 #!/bin/bash
-# Verify an RFS flist (.fl) by inspecting, listing tree, and optional mount test.
+# Verify an RFS flist manifest: inspect, tree, and optional mount test (best-effort)
-# Usage:
+# This script is safe to run on developer machines; mount test may require root and FUSE policy.
-#   ./scripts/rfs/verify-flist.sh /path/to/foo.fl
-#   ./scripts/rfs/verify-flist.sh /path/to/foo.fl --mount
-#
-# Notes:
-# - Requires the rfs binary (PATH or components fallback).
-# - --mount performs a temporary mount (needs FUSE and privileges).
 
 set -euo pipefail
 
@@ -14,52 +8,103 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${HERE}/common.sh"
 
+usage() {
+  cat <<'EOF'
+Usage: scripts/rfs/verify-flist.sh -m path/to/manifest.fl [--tree] [--mount] [--mountpoint DIR]
+
+Actions:
+  - Inspect manifest metadata (always)
+  - Tree view of entries (--tree)
+  - Optional mount test (--mount) to a temporary or given mountpoint
+
+Notes:
+  - Mount test typically requires root and proper FUSE configuration.
+  - On success, a quick directory listing is shown, then the mount is unmounted.
+EOF
+}
+
 section() { echo -e "\n==== $* ====\n"; }
 
-if [[ $# -lt 1 ]]; then
+MANIFEST=""
-  echo "Usage: $0 /path/to/foo.fl [--mount]"
+DO_TREE=0
+DO_MOUNT=0
+MOUNTPOINT=""
+
+# Parse args
+if [[ $# -eq 0 ]]; then
+  usage
   exit 1
 fi
 
-FL_PATH="$1"
+while [[ $# -gt 0 ]]; do
-DO_MOUNT="${2:-}"
+  case "$1" in
+    -m|--manifest)
+      MANIFEST="${2:-}"; shift 2;;
+    --tree)
+      DO_TREE=1; shift;;
+    --mount)
+      DO_MOUNT=1; shift;;
+    --mountpoint)
+      MOUNTPOINT="${2:-}"; shift 2;;
+    -h|--help)
+      usage; exit 0;;
+    *)
+      echo "Unknown argument: $1" >&2; usage; exit 1;;
+  esac
+done
 
-if [[ ! -f "$FL_PATH" ]]; then
+if [[ -z "${MANIFEST}" || ! -f "${MANIFEST}" ]]; then
-  echo "[ERROR] flist not found: ${FL_PATH}" >&2
+  log_error "Manifest not found: ${MANIFEST:-<empty>}"
   exit 1
 fi
 
-section "Locating rfs binary"
+# Ensure rfs binary is available
 rfs_common_locate_rfs
 
-section "Inspect flist"
+section "Inspecting manifest"
-safe_execute "${RFS_BIN}" flist inspect "${FL_PATH}" || true
+safe_execute "${RFS_BIN}" flist inspect -m "${MANIFEST}" || {
-
+  log_warn "rfs flist inspect failed (old rfs?)"
-section "Tree (first 100 entries)"
+  log_warn "Try: ${RFS_BIN} inspect ${MANIFEST}"
-safe_execute "${RFS_BIN}" flist tree "${FL_PATH}" | head -n 100 || true
-
-if [[ "$DO_MOUNT" == "--mount" ]]; then
-  section "Attempting temporary mount"
-  MNT="$(mktemp -d /tmp/rfs-mnt-XXXXXX)"
-  cleanup() {
-    set +e
-    if mountpoint -q "${MNT}" 2>/dev/null; then
-      echo "[INFO] Unmounting ${MNT}"
-      fusermount -u "${MNT}" 2>/dev/null || umount "${MNT}" 2>/dev/null || true
-    fi
-    rmdir "${MNT}" 2>/dev/null || true
 }
-  trap cleanup EXIT INT TERM
 
-  echo "[INFO] Mountpoint: ${MNT}"
+if [[ ${DO_TREE} -eq 1 ]]; then
-  # Try mount; some environments require sudo or have limited FUSE. Best-effort.
+  section "Listing manifest tree"
-  if "${RFS_BIN}" mount -m "${FL_PATH}" "${MNT}"; then
+  safe_execute "${RFS_BIN}" flist tree -m "${MANIFEST}" 2>/dev/null || {
-    echo "[INFO] Mounted. Listing top-level entries:"
+    log_warn "rfs flist tree failed; attempting fallback 'tree'"
-    ls -la "${MNT}" | head -n 50 || true
+    safe_execute "${RFS_BIN}" tree -m "${MANIFEST}" || true
+  }
+fi
+
+if [[ ${DO_MOUNT} -eq 1 ]]; then
+  section "Mount test"
+  if [[ "$(id -u)" -ne 0 ]]; then
+    log_warn "Mount test skipped: requires root (uid 0)"
   else
-    echo "[WARN] rfs mount failed (FUSE/permissions?). Skipping mount verification." >&2
+    # Decide mountpoint
+    local_mp_created=0
+    if [[ -z "${MOUNTPOINT}" ]]; then
+      MOUNTPOINT="$(mktemp -d /tmp/rfs-mnt.XXXXXX)"
+      local_mp_created=1
+    else
+      mkdir -p "${MOUNTPOINT}"
+    fi
+    log_info "Mounting ${MANIFEST} -> ${MOUNTPOINT}"
+    set +e
+    # Best-effort background mount
+    (setsid "${RFS_BIN}" mount -m "${MANIFEST}" "${MOUNTPOINT}" >/tmp/rfs-mount.log 2>&1) &
+    mpid=$!
+    sleep 2
+    ls -la "${MOUNTPOINT}" | head -n 50 || true
+    # Try to unmount and stop background process
+    umount "${MOUNTPOINT}" 2>/dev/null || fusermount -u "${MOUNTPOINT}" 2>/dev/null || true
+    kill "${mpid}" 2>/dev/null || true
+    set -e
+    if [[ ${local_mp_created} -eq 1 ]]; then
+      rmdir "${MOUNTPOINT}" 2>/dev/null || true
+    fi
+    log_info "Mount test done (see /tmp/rfs-mount.log if issues)"
   fi
 fi
 
-section "Done"
+section "Verify complete"
-echo "[INFO] Verified flist: ${FL_PATH}"
+log_info "Manifest: ${MANIFEST}"