From cf05e0ca5b0820327a55f92303508a140020ff0b Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Thu, 2 Oct 2025 17:41:16 +0200 Subject: [PATCH] rfs: add pack-tree.sh to pack arbitrary directory to flist using config/rfs.conf; enable --debug on rfs pack for verbose diagnostics --- scripts/rfs/pack-tree.sh | 155 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100755 scripts/rfs/pack-tree.sh diff --git a/scripts/rfs/pack-tree.sh b/scripts/rfs/pack-tree.sh new file mode 100755 index 0000000..6d48ee7 --- /dev/null +++ b/scripts/rfs/pack-tree.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# Pack an arbitrary directory tree into an RFS flist and upload blobs to S3 (Garage) +# - Uses config from config/rfs.conf (or rfs.conf.example fallback) +# - Packs the specified directory (default: repository root) +# - Patches manifest route URL with read-only S3 creds +# - Optionally patches stores to WEB_ENDPOINT and uploads the .fl via MinIO client +# +# Usage: +# scripts/rfs/pack-tree.sh [-p PATH] [-n MANIFEST_BASENAME] [--web-endpoint URL] [--keep-s3-fallback] [--no-upload] +# Examples: +# scripts/rfs/pack-tree.sh +# scripts/rfs/pack-tree.sh -p ./components/zinit -n zinit-src +# WEB_ENDPOINT=https://hub.grid.tf/zos/zosbuilder/store scripts/rfs/pack-tree.sh -p dist --keep-s3-fallback + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${HERE}/common.sh" + +section() { echo -e "\n==== $* ====\n"; } + +print_help() { + cat <- + --web-endpoint URL Override WEB_ENDPOINT for stores patching (default from rfs.conf) + --keep-s3-fallback Keep s3:// store rows alongside HTTPS store in manifest + --no-upload Do not upload the .fl manifest via MinIO client even if enabled in config + -h, --help Show this help + +Environment/config: + Reads S3 and route settings from config/rfs.conf (or rfs.conf.example). + Honors: WEB_ENDPOINT, KEEP_S3_FALLBACK, UPLOAD_MANIFESTS. +EOF +} + +# Defaults +SRC_PATH="" +MANIFEST_NAME="" +ARG_WEB_ENDPOINT="" +ARG_KEEP_S3="false" +ARG_NO_UPLOAD="false" + +# Parse args +while [[ $# -gt 0 ]]; do + case "$1" in + -p|--path) + SRC_PATH="$2"; shift 2;; + -n|--name) + MANIFEST_NAME="$2"; shift 2;; + --web-endpoint) + ARG_WEB_ENDPOINT="$2"; shift 2;; + --keep-s3-fallback) + ARG_KEEP_S3="true"; shift 1;; + --no-upload) + ARG_NO_UPLOAD="true"; shift 1;; + -h|--help) + print_help; exit 0;; + *) + echo "Unknown argument: $1" >&2; print_help; exit 1;; + esac +done + +# Determine PROJECT_ROOT and defaults +PROJECT_ROOT="${PROJECT_ROOT:-$(rfs_common_project_root)}" +if [[ -z "${SRC_PATH}" ]]; then + SRC_PATH="${PROJECT_ROOT}" +fi + +# Normalize SRC_PATH to absolute +if [[ "${SRC_PATH}" != /* ]]; then + SRC_PATH="$(cd "${SRC_PATH}" && pwd)" +fi +if [[ ! -d "${SRC_PATH}" ]]; then + log_error "Source path is not a directory: ${SRC_PATH}" + exit 1 +fi + +# Compute default manifest name if not given +if [[ -z "${MANIFEST_NAME}" ]]; then + base="$(basename "${SRC_PATH}")" + ts="$(date -u +%Y%m%d%H%M%S)" + MANIFEST_NAME="tree-${base}-${ts}" +fi +MANIFEST_FILE="${MANIFEST_NAME%.fl}.fl" + +section "Loading RFS and kernel configuration" +# Kernel version for consistent logs (not strictly required for generic pack) +rfs_common_load_build_kernel_version +rfs_common_load_rfs_s3_config +# Allow CLI override for WEB_ENDPOINT and KEEP_S3_FALLBACK +if [[ -n "${ARG_WEB_ENDPOINT}" ]]; then + WEB_ENDPOINT="${ARG_WEB_ENDPOINT}" +fi +if [[ "${ARG_KEEP_S3}" == "true" ]]; then + KEEP_S3_FALLBACK="true" +fi +rfs_common_build_s3_store_uri +rfs_common_locate_rfs + +# Prepare output +MANIFEST_PATH="$(rfs_common_prepare_output "${MANIFEST_FILE}")" + +section "Packing directory to flist" +log_info "Source path: ${SRC_PATH}" +log_info "Manifest: ${MANIFEST_PATH}" +log_info "Store: ${RFS_S3_STORE_URI}" +safe_execute "${RFS_BIN}" pack --debug -m "${MANIFEST_PATH}" -s "${RFS_S3_STORE_URI}" "${SRC_PATH}" + +section "Patching route.url in manifest to S3 read-only endpoint" +rfs_common_build_route_url +rfs_common_patch_flist_route_url "${MANIFEST_PATH}" + +if [[ -n "${WEB_ENDPOINT:-}" ]]; then + section "Patching stores to HTTPS web endpoint" + log_info "WEB_ENDPOINT=${WEB_ENDPOINT}" + log_info "KEEP_S3_FALLBACK=${KEEP_S3_FALLBACK:-false}" + rfs_common_patch_flist_stores "${MANIFEST_PATH}" "${WEB_ENDPOINT}" "${KEEP_S3_FALLBACK:-false}" +else + log_warn "WEB_ENDPOINT not set; leaving manifest stores as-is (s3:// only)" +fi + +# Optional manifest upload via MinIO client +UPLOAD="${UPLOAD_MANIFESTS:-false}" +if [[ "${ARG_NO_UPLOAD}" == "true" ]]; then + UPLOAD="false" +fi + +if [[ "${UPLOAD}" == "true" ]]; then + section "Uploading manifest .fl via MinIO client (mcli/mc)" + if command -v mcli >/dev/null 2>&1; then + MCLI_BIN="mcli" + elif command -v mc >/dev/null 2>&1; then + MCLI_BIN="mc" + else + log_warn "MinIO Client not found (expected mcli or mc); skipping manifest upload" + MCLI_BIN="" + fi + if [[ -n "${MCLI_BIN}" ]]; then + local_subpath="${MANIFESTS_SUBPATH:-manifests}" + safe_execute "${MCLI_BIN}" alias set rfs "${S3_ENDPOINT}" "${S3_ACCESS_KEY}" "${S3_SECRET_KEY}" + dst="rfs/${S3_BUCKET}/${S3_PREFIX%/}/${local_subpath%/}/${MANIFEST_FILE}" + log_info "${MCLI_BIN} cp ${MANIFEST_PATH} ${dst}" + safe_execute "${MCLI_BIN}" cp "${MANIFEST_PATH}" "${dst}" + fi +else + log_info "UPLOAD_MANIFESTS=false; skipping manifest upload" +fi + +section "Done" +log_info "Packed: ${MANIFEST_PATH}" \ No newline at end of file