From 44cbf20d7baef06cf1a743747ef08d136b242a04 Mon Sep 17 00:00:00 2001 From: despiegk Date: Wed, 9 Apr 2025 07:54:37 +0200 Subject: [PATCH] ... --- doctree/Cargo.toml | 5 + doctree/src/collection.rs | 39 +- doctree/src/doctree.rs | 270 +++++++++++- doctree/src/error.rs | 4 + doctree/src/lib.rs | 4 +- doctree/src/storage.rs | 232 +++++++++-- doctree/src/utils.rs | 17 +- doctree_implementation_plan.md | 657 ++++++++++-------------------- doctreecmd/src/main.rs | 291 ++++++++++++- example_commands.sh | 31 ++ examples/grid1/.collection | 1 + examples/parent/docs2/.collection | 1 + runexample.sh | 39 ++ 13 files changed, 1073 insertions(+), 518 deletions(-) create mode 100755 example_commands.sh create mode 100755 runexample.sh diff --git a/doctree/Cargo.toml b/doctree/Cargo.toml index 387fa29..7108d38 100644 --- a/doctree/Cargo.toml +++ b/doctree/Cargo.toml @@ -8,3 +8,8 @@ walkdir = "2.3.3" pulldown-cmark = "0.9.3" thiserror = "1.0.40" lazy_static = "1.4.0" +toml = "0.7.3" +serde = { version = "1.0", features = ["derive"] } +redis = { version = "0.23.0", features = ["tokio-comp"] } +tokio = { version = "1.28.0", features = ["full"] } +sal = { git = "https://git.ourworld.tf/herocode/sal.git", branch = "main" } diff --git a/doctree/src/collection.rs b/doctree/src/collection.rs index 28577ad..5ccff8e 100644 --- a/doctree/src/collection.rs +++ b/doctree/src/collection.rs @@ -58,7 +58,10 @@ impl Collection { /// /// Ok(()) on success or an error pub fn scan(&self) -> Result<()> { + println!("DEBUG: Scanning collection '{}' at path {:?}", self.name, self.path); + // Delete existing collection data if any + println!("DEBUG: Deleting existing collection data from Redis key 'collections:{}'", self.name); self.storage.delete_collection(&self.name)?; // Walk through the directory @@ -79,6 +82,12 @@ impl Collection { continue; } + // Skip files that start with a dot (.) + let file_name = entry.file_name().to_string_lossy(); + if file_name.starts_with(".") { + continue; + } + // Get the relative path from the base path let rel_path = match entry.path().strip_prefix(&self.path) { Ok(path) => path, @@ -93,11 +102,30 @@ impl Collection { let filename = entry.file_name().to_string_lossy().to_string(); let namefixed_filename = name_fix(&filename); + // Determine if this is a document (markdown file) or an image + let is_markdown = filename.to_lowercase().ends_with(".md"); + let is_image = filename.to_lowercase().ends_with(".png") || + filename.to_lowercase().ends_with(".jpg") || + filename.to_lowercase().ends_with(".jpeg") || + filename.to_lowercase().ends_with(".gif") || + filename.to_lowercase().ends_with(".svg"); + + let file_type = if is_markdown { + "document" + } else if is_image { + "image" + } else { + "file" + }; + // Store in Redis using the namefixed filename as the key // Store the original relative path to preserve case and special characters + println!("DEBUG: Storing {} '{}' in Redis key 'collections:{}' with key '{}' and value '{}'", + file_type, filename, self.name, namefixed_filename, rel_path.to_string_lossy()); + self.storage.store_collection_entry( - &self.name, - &namefixed_filename, + &self.name, + &namefixed_filename, &rel_path.to_string_lossy() )?; } @@ -125,6 +153,13 @@ impl Collection { let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_page_name) .map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?; + // Check if the path is valid + if self.path.as_os_str().is_empty() { + // If the path is empty, we're working with a collection loaded from Redis + // Return a placeholder content for demonstration purposes + return Ok(format!("Content for {} in collection {}\nThis is a placeholder since the actual file path is not available.", page_name, self.name)); + } + // Read the file let full_path = self.path.join(rel_path); let content = fs::read_to_string(full_path) diff --git a/doctree/src/doctree.rs b/doctree/src/doctree.rs index afa1bc9..a98005d 100644 --- a/doctree/src/doctree.rs +++ b/doctree/src/doctree.rs @@ -1,6 +1,8 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; +use std::fs; +use serde::Deserialize; use crate::collection::{Collection, CollectionBuilder}; use crate::error::{DocTreeError, Result}; @@ -8,6 +10,14 @@ use crate::storage::RedisStorage; use crate::include::process_includes; use crate::utils::{name_fix, ensure_md_extension}; +/// Configuration for a collection from a .collection file +#[derive(Deserialize, Default, Debug)] +struct CollectionConfig { + /// Optional name of the collection + name: Option, + // Add other configuration options as needed +} + // Global variable to track the current collection name // This is for compatibility with the Go implementation lazy_static::lazy_static! { @@ -144,13 +154,100 @@ impl DocTree { Ok(()) } + /// Delete all collections from the DocTree and Redis + /// + /// # Returns + /// + /// Ok(()) on success or an error + pub fn delete_all_collections(&mut self) -> Result<()> { + // Delete all collections from Redis + self.storage.delete_all_collections()?; + + // Clear the collections map + self.collections.clear(); + + // Reset the default collection + self.default_collection = None; + + Ok(()) + } + /// List all collections /// /// # Returns /// /// A vector of collection names pub fn list_collections(&self) -> Vec { - self.collections.keys().cloned().collect() + // First, try to get collections from the in-memory map + let mut collections = self.collections.keys().cloned().collect::>(); + + // If no collections are found, try to get them from Redis + if collections.is_empty() { + // Get all collection keys from Redis + if let Ok(keys) = self.storage.list_all_collections() { + collections = keys; + } + } + + collections + } + + /// Load a collection from Redis + /// + /// # Arguments + /// + /// * `name` - Name of the collection + /// + /// # Returns + /// + /// Ok(()) on success or an error + pub fn load_collection(&mut self, name: &str) -> Result<()> { + // Check if the collection exists in Redis + if !self.storage.collection_exists(name)? { + return Err(DocTreeError::CollectionNotFound(name.to_string())); + } + + // Create a new collection + let collection = Collection { + path: PathBuf::new(), // We don't have the path, but it's not needed for Redis operations + name: name.to_string(), + storage: self.storage.clone(), + }; + + // Add to the collections map + self.collections.insert(name.to_string(), collection); + + Ok(()) + } + + /// Load all collections from Redis + /// + /// # Returns + /// + /// Ok(()) on success or an error + pub fn load_collections_from_redis(&mut self) -> Result<()> { + // Get all collection names from Redis + let collections = self.storage.list_all_collections()?; + + // Load each collection + for name in collections { + // Skip if already loaded + if self.collections.contains_key(&name) { + continue; + } + + // Create a new collection + let collection = Collection { + path: PathBuf::new(), // We don't have the path, but it's not needed for Redis operations + name: name.clone(), + storage: self.storage.clone(), + }; + + // Add to the collections map + self.collections.insert(name, collection); + } + + Ok(()) } /// Get a page by name from a specific collection @@ -163,7 +260,7 @@ impl DocTree { /// # Returns /// /// The page content or an error - pub fn page_get(&self, collection_name: Option<&str>, page_name: &str) -> Result { + pub fn page_get(&mut self, collection_name: Option<&str>, page_name: &str) -> Result { let (collection_name, page_name) = self.resolve_collection_and_page(collection_name, page_name)?; // Get the collection @@ -293,6 +390,111 @@ impl DocTree { } } } + + /// Recursively scan directories for .collection files and add them as collections + /// + /// # Arguments + /// + /// * `root_path` - The root path to start scanning from + /// + /// # Returns + /// + /// Ok(()) on success or an error + pub fn scan_collections>(&mut self, root_path: P) -> Result<()> { + let root_path = root_path.as_ref(); + + println!("DEBUG: Scanning for collections in directory: {:?}", root_path); + + // Walk through the directory tree + for entry in walkdir::WalkDir::new(root_path).follow_links(true) { + let entry = match entry { + Ok(entry) => entry, + Err(e) => { + eprintln!("Error walking directory: {}", e); + continue; + } + }; + + // Skip directories and files that start with a dot (.) + let file_name = entry.file_name().to_string_lossy(); + if file_name.starts_with(".") { + continue; + } + + // Skip non-directories + if !entry.file_type().is_dir() { + continue; + } + + // Check if this directory contains a .collection file + let collection_file_path = entry.path().join(".collection"); + if collection_file_path.exists() { + // Found a collection directory + println!("DEBUG: Found .collection file at: {:?}", collection_file_path); + let dir_path = entry.path(); + + // Get the directory name as a fallback collection name + let dir_name = dir_path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or("unnamed"); + + // Try to read and parse the .collection file + let collection_name = match fs::read_to_string(&collection_file_path) { + Ok(content) => { + if content.trim().is_empty() { + // Empty file, use directory name (name_fixed) + dir_name.to_string() // We'll apply name_fix later at line 372 + } else { + // Parse as TOML + match toml::from_str::(&content) { + Ok(config) => { + // Use the name from config if available, otherwise use directory name + config.name.unwrap_or_else(|| dir_name.to_string()) + }, + Err(e) => { + eprintln!("Error parsing .collection file at {:?}: {}", collection_file_path, e); + dir_name.to_string() + } + } + } + }, + Err(e) => { + eprintln!("Error reading .collection file at {:?}: {}", collection_file_path, e); + dir_name.to_string() + } + }; + + // Apply name_fix to the collection name + let namefixed_collection_name = name_fix(&collection_name); + + // Add the collection to the DocTree + println!("DEBUG: Adding collection '{}' from directory {:?}", namefixed_collection_name, dir_path); + match self.add_collection(dir_path, &namefixed_collection_name) { + Ok(collection) => { + println!("DEBUG: Successfully added collection '{}' from {:?}", namefixed_collection_name, dir_path); + println!("DEBUG: Collection stored in Redis key 'collections:{}'", collection.name); + + // Count documents and images + let docs = collection.page_list().unwrap_or_default(); + let files = collection.file_list().unwrap_or_default(); + let images = files.iter().filter(|f| + f.ends_with(".png") || f.ends_with(".jpg") || + f.ends_with(".jpeg") || f.ends_with(".gif") || + f.ends_with(".svg") + ).count(); + + println!("DEBUG: Collection '{}' contains {} documents and {} images", + namefixed_collection_name, docs.len(), images); + }, + Err(e) => { + eprintln!("Error adding collection '{}' from {:?}: {}", namefixed_collection_name, dir_path, e); + } + } + } + } + + Ok(()) + } } impl DocTreeBuilder { @@ -363,6 +565,47 @@ impl DocTreeBuilder { self } + /// Scan for collections in the given root path + /// + /// # Arguments + /// + /// * `root_path` - The root path to scan for collections + /// + /// # Returns + /// + /// Self for method chaining or an error + pub fn scan_collections>(self, root_path: P) -> Result { + // Ensure storage is set + let storage = self.storage.as_ref().ok_or_else(|| { + DocTreeError::MissingParameter("storage".to_string()) + })?; + + // Create a temporary DocTree to scan collections + let mut temp_doctree = DocTree { + collections: HashMap::new(), + default_collection: None, + storage: storage.clone(), + name: self.name.clone().unwrap_or_default(), + path: self.path.clone().unwrap_or_else(|| PathBuf::from("")), + }; + + // Scan for collections + temp_doctree.scan_collections(root_path)?; + + // Create a new builder with the scanned collections + let mut new_builder = self; + for (name, collection) in temp_doctree.collections { + new_builder.collections.insert(name.clone(), collection); + + // If no default collection is set, use the first one found + if new_builder.default_collection.is_none() { + new_builder.default_collection = Some(name); + } + } + + Ok(new_builder) + } + /// Build the DocTree /// /// # Returns @@ -375,7 +618,7 @@ impl DocTreeBuilder { })?; // Create the DocTree - let doctree = DocTree { + let mut doctree = DocTree { collections: self.collections, default_collection: self.default_collection, storage: storage.clone(), @@ -389,6 +632,9 @@ impl DocTreeBuilder { *current_collection_name = Some(default_collection.clone()); } + // Load all collections from Redis + doctree.load_collections_from_redis()?; + Ok(doctree) } } @@ -430,4 +676,22 @@ pub fn new>(args: &[&str]) -> Result { } builder.build() +} + +/// Create a new DocTree by scanning a directory for collections +/// +/// # Arguments +/// +/// * `root_path` - The root path to scan for collections +/// +/// # Returns +/// +/// A new DocTree or an error +pub fn from_directory>(root_path: P) -> Result { + let storage = RedisStorage::new("redis://localhost:6379")?; + + DocTree::builder() + .with_storage(storage) + .scan_collections(root_path)? + .build() } \ No newline at end of file diff --git a/doctree/src/error.rs b/doctree/src/error.rs index bdac532..02a1fd4 100644 --- a/doctree/src/error.rs +++ b/doctree/src/error.rs @@ -38,6 +38,10 @@ pub enum DocTreeError { /// Missing required parameter #[error("Missing required parameter: {0}")] MissingParameter(String), + + /// Redis error + #[error("Redis error: {0}")] + RedisError(String), } /// Result type alias for doctree operations diff --git a/doctree/src/lib.rs b/doctree/src/lib.rs index 2989d40..7acdc78 100644 --- a/doctree/src/lib.rs +++ b/doctree/src/lib.rs @@ -3,7 +3,7 @@ //! It provides functionality for scanning directories, managing collections, //! and processing includes between documents. -// Import lazy_static +// Import lazy_static for global state #[macro_use] extern crate lazy_static; @@ -17,7 +17,7 @@ mod include; pub use error::{DocTreeError, Result}; pub use storage::RedisStorage; pub use collection::{Collection, CollectionBuilder}; -pub use doctree::{DocTree, DocTreeBuilder, new}; +pub use doctree::{DocTree, DocTreeBuilder, new, from_directory}; pub use include::process_includes; #[cfg(test)] diff --git a/doctree/src/storage.rs b/doctree/src/storage.rs index f588703..71493b1 100644 --- a/doctree/src/storage.rs +++ b/doctree/src/storage.rs @@ -1,12 +1,13 @@ -use std::collections::HashMap; +use redis::{Client, Commands, Connection}; use std::sync::{Arc, Mutex}; use crate::error::{DocTreeError, Result}; /// Storage backend for doctree pub struct RedisStorage { - // Using a simple in-memory storage for demonstration - // In a real implementation, this would be a Redis client - collections: Arc>>>, + // Redis client + client: Client, + // Connection pool + connection: Arc>, } impl RedisStorage { @@ -20,9 +21,16 @@ impl RedisStorage { /// # Returns /// /// A new RedisStorage instance or an error - pub fn new(_url: &str) -> Result { + pub fn new(url: &str) -> Result { + // Create a Redis client + let client = Client::open(url).map_err(|e| DocTreeError::RedisError(format!("Failed to connect to Redis: {}", e)))?; + + // Get a connection + let connection = client.get_connection().map_err(|e| DocTreeError::RedisError(format!("Failed to get Redis connection: {}", e)))?; + Ok(Self { - collections: Arc::new(Mutex::new(HashMap::new())), + client, + connection: Arc::new(Mutex::new(connection)), }) } @@ -38,15 +46,21 @@ impl RedisStorage { /// /// Ok(()) on success or an error pub fn store_collection_entry(&self, collection: &str, key: &str, value: &str) -> Result<()> { - let mut collections = self.collections.lock().unwrap(); + let redis_key = format!("collections:{}", collection); + println!("DEBUG: Redis operation - HSET {} {} {}", redis_key, key, value); - // Get or create the collection - let collection_entries = collections - .entry(format!("collections:{}", collection)) - .or_insert_with(HashMap::new); + // Get a connection from the pool + let mut conn = self.connection.lock().unwrap(); - // Store the entry - collection_entries.insert(key.to_string(), value.to_string()); + // Store the entry using HSET + redis::cmd("HSET") + .arg(&redis_key) + .arg(key) + .arg(value) + .execute(&mut *conn); + + println!("DEBUG: Stored entry in Redis - collection: '{}', key: '{}', value: '{}'", + collection, key, value); Ok(()) } @@ -62,17 +76,32 @@ impl RedisStorage { /// /// The entry value or an error pub fn get_collection_entry(&self, collection: &str, key: &str) -> Result { - let collections = self.collections.lock().unwrap(); - - // Get the collection let collection_key = format!("collections:{}", collection); - let collection_entries = collections.get(&collection_key) - .ok_or_else(|| DocTreeError::CollectionNotFound(collection.to_string()))?; + println!("DEBUG: Redis operation - HGET {} {}", collection_key, key); - // Get the entry - collection_entries.get(key) - .cloned() - .ok_or_else(|| DocTreeError::FileNotFound(key.to_string())) + // Get a connection from the pool + let mut conn = self.connection.lock().unwrap(); + + // Get the entry using HGET + let result: Option = redis::cmd("HGET") + .arg(&collection_key) + .arg(key) + .query(&mut *conn) + .map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?; + + // Check if the entry exists + match result { + Some(value) => { + println!("DEBUG: Retrieved entry from Redis - collection: '{}', key: '{}', value: '{}'", + collection, key, value); + Ok(value) + }, + None => { + println!("DEBUG: Entry not found in Redis - collection: '{}', key: '{}'", + collection, key); + Err(DocTreeError::FileNotFound(key.to_string())) + } + } } /// Delete a collection entry @@ -86,15 +115,30 @@ impl RedisStorage { /// /// Ok(()) on success or an error pub fn delete_collection_entry(&self, collection: &str, key: &str) -> Result<()> { - let mut collections = self.collections.lock().unwrap(); - - // Get the collection let collection_key = format!("collections:{}", collection); - let collection_entries = collections.get_mut(&collection_key) - .ok_or_else(|| DocTreeError::CollectionNotFound(collection.to_string()))?; + println!("DEBUG: Redis operation - HDEL {} {}", collection_key, key); - // Remove the entry - collection_entries.remove(key); + // Get a connection from the pool + let mut conn = self.connection.lock().unwrap(); + + // Delete the entry using HDEL + let exists: bool = redis::cmd("HEXISTS") + .arg(&collection_key) + .arg(key) + .query(&mut *conn) + .map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?; + + if !exists { + return Err(DocTreeError::CollectionNotFound(collection.to_string())); + } + + redis::cmd("HDEL") + .arg(&collection_key) + .arg(key) + .execute(&mut *conn); + + println!("DEBUG: Deleted entry from Redis - collection: '{}', key: '{}'", + collection, key); Ok(()) } @@ -109,15 +153,30 @@ impl RedisStorage { /// /// A vector of entry keys or an error pub fn list_collection_entries(&self, collection: &str) -> Result> { - let collections = self.collections.lock().unwrap(); - - // Get the collection let collection_key = format!("collections:{}", collection); - let collection_entries = collections.get(&collection_key) - .ok_or_else(|| DocTreeError::CollectionNotFound(collection.to_string()))?; + println!("DEBUG: Redis operation - HKEYS {}", collection_key); - // Get the keys - let keys = collection_entries.keys().cloned().collect(); + // Get a connection from the pool + let mut conn = self.connection.lock().unwrap(); + + // Check if the collection exists + let exists: bool = redis::cmd("EXISTS") + .arg(&collection_key) + .query(&mut *conn) + .map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?; + + if !exists { + return Err(DocTreeError::CollectionNotFound(collection.to_string())); + } + + // Get all keys using HKEYS + let keys: Vec = redis::cmd("HKEYS") + .arg(&collection_key) + .query(&mut *conn) + .map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?; + + println!("DEBUG: Listed {} entries from Redis - collection: '{}'", + keys.len(), collection); Ok(keys) } @@ -132,10 +191,18 @@ impl RedisStorage { /// /// Ok(()) on success or an error pub fn delete_collection(&self, collection: &str) -> Result<()> { - let mut collections = self.collections.lock().unwrap(); + let redis_key = format!("collections:{}", collection); + println!("DEBUG: Redis operation - DEL {}", redis_key); - // Remove the collection - collections.remove(&format!("collections:{}", collection)); + // Get a connection from the pool + let mut conn = self.connection.lock().unwrap(); + + // Delete the collection using DEL + redis::cmd("DEL") + .arg(&redis_key) + .execute(&mut *conn); + + println!("DEBUG: Deleted collection from Redis - collection: '{}'", collection); Ok(()) } @@ -150,20 +217,99 @@ impl RedisStorage { /// /// true if the collection exists, false otherwise pub fn collection_exists(&self, collection: &str) -> Result { - let collections = self.collections.lock().unwrap(); + let collection_key = format!("collections:{}", collection); + println!("DEBUG: Redis operation - EXISTS {}", collection_key); - // Check if the collection exists - let exists = collections.contains_key(&format!("collections:{}", collection)); + // Get a connection from the pool + let mut conn = self.connection.lock().unwrap(); + + // Check if the collection exists using EXISTS + let exists: bool = redis::cmd("EXISTS") + .arg(&collection_key) + .query(&mut *conn) + .map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?; + + println!("DEBUG: Collection exists check - collection: '{}', exists: {}", + collection, exists); Ok(exists) } + + /// List all collections in Redis + /// + /// # Returns + /// + /// A vector of collection names or an error + pub fn list_all_collections(&self) -> Result> { + println!("DEBUG: Redis operation - KEYS collections:*"); + + // Get a connection from the pool + let mut conn = self.connection.lock().unwrap(); + + // Get all collection keys + let keys: Vec = redis::cmd("KEYS") + .arg("collections:*") + .query(&mut *conn) + .map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?; + + // Extract collection names from keys (remove the "collections:" prefix) + let collections = keys.iter() + .filter_map(|key| { + if key.starts_with("collections:") { + Some(key[12..].to_string()) + } else { + None + } + }) + .collect(); + + println!("DEBUG: Found {} collections in Redis", keys.len()); + + Ok(collections) + } + + /// Delete all collections from Redis + /// + /// # Returns + /// + /// Ok(()) on success or an error + pub fn delete_all_collections(&self) -> Result<()> { + println!("DEBUG: Redis operation - KEYS collections:*"); + + // Get a connection from the pool + let mut conn = self.connection.lock().unwrap(); + + // Get all collection keys + let keys: Vec = redis::cmd("KEYS") + .arg("collections:*") + .query(&mut *conn) + .map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?; + + println!("DEBUG: Found {} collections in Redis", keys.len()); + + // Delete each collection + for key in keys { + println!("DEBUG: Redis operation - DEL {}", key); + redis::cmd("DEL") + .arg(&key) + .execute(&mut *conn); + println!("DEBUG: Deleted collection from Redis - key: '{}'", key); + } + + Ok(()) + } } // Implement Clone for RedisStorage impl Clone for RedisStorage { fn clone(&self) -> Self { + // Create a new connection + let connection = self.client.get_connection() + .expect("Failed to get Redis connection"); + Self { - collections: Arc::clone(&self.collections), + client: self.client.clone(), + connection: Arc::new(Mutex::new(connection)), } } } \ No newline at end of file diff --git a/doctree/src/utils.rs b/doctree/src/utils.rs index 3c20c10..5c1c296 100644 --- a/doctree/src/utils.rs +++ b/doctree/src/utils.rs @@ -1,5 +1,6 @@ use pulldown_cmark::{Parser, Options, html}; use std::path::Path; +use sal::text; /// Fix a name to be used as a key /// @@ -13,19 +14,9 @@ use std::path::Path; /// # Returns /// /// The fixed name -pub fn name_fix(name: &str) -> String { - // Convert to lowercase - let mut result = name.to_lowercase(); - - // Replace spaces with hyphens - result = result.replace(' ', "-"); - - // Remove special characters - result = result.chars() - .filter(|c| c.is_alphanumeric() || *c == '-' || *c == '.') - .collect(); - - result +pub fn name_fix(text: &str) -> String { + // Use the name_fix function from the SAL library + text::name_fix(text) } /// Convert markdown to HTML diff --git a/doctree_implementation_plan.md b/doctree_implementation_plan.md index 34e9bfc..317d637 100644 --- a/doctree_implementation_plan.md +++ b/doctree_implementation_plan.md @@ -1,499 +1,258 @@ -# DocTree Implementation Plan +# Implementation Plan: DocTree Collection Scanner ## Overview -The DocTree library will be a Rust implementation of the Go reference, maintaining the core functionality while improving the API design to be more idiomatic Rust. We'll use Redis as the storage backend and implement a minimal CLI example to demonstrate usage. +We need to expand the doctree library to: +1. Add a recursive scan function to the DocTree struct +2. Detect directories containing `.collection` files +3. Parse `.collection` files as TOML to extract collection names +4. Replace the current `name_fix` function with the one from the sal library +5. Populate collections with all files found under the collection directories -## Architecture +## Detailed Implementation Plan -```mermaid -classDiagram - class DocTree { - +collections: HashMap - +default_collection: Option - +new() DocTreeBuilder - +add_collection(path, name) Result<&Collection> - +get_collection(name) Result<&Collection> - +delete_collection(name) Result<()> - +list_collections() Vec - +page_get(collection, page) Result - +page_get_html(collection, page) Result - +file_get_url(collection, file) Result - } +### 1. Update Dependencies - class DocTreeBuilder { - -collections: HashMap - -default_collection: Option - +with_collection(path, name) DocTreeBuilder - +with_default_collection(name) DocTreeBuilder - +build() Result - } +First, we need to add the necessary dependencies to the Cargo.toml file: - class Collection { - +path: String - +name: String - +new(path, name) CollectionBuilder - +scan() Result<()> - +page_get(name) Result - +page_set(name, content) Result<()> - +page_delete(name) Result<()> - +page_list() Result> - +file_get_url(name) Result - +file_set(name, content) Result<()> - +file_delete(name) Result<()> - +file_list() Result> - +page_get_html(name) Result - } - - class CollectionBuilder { - -path: String - -name: String - +build() Result - } - - class RedisStorage { - +client: redis::Client - +new(url) Result - +store_collection_entry(collection, key, value) Result<()> - +get_collection_entry(collection, key) Result - +delete_collection_entry(collection, key) Result<()> - +list_collection_entries(collection) Result> - +delete_collection(collection) Result<()> - } - - class IncludeProcessor { - +process_includes(content, collection, doctree) Result - } - - DocTree --> DocTreeBuilder : creates - DocTree --> "0..*" Collection : contains - Collection --> CollectionBuilder : creates - DocTree --> RedisStorage : uses - Collection --> RedisStorage : uses - DocTree --> IncludeProcessor : uses +```toml +[dependencies] +walkdir = "2.3.3" +pulldown-cmark = "0.9.3" +thiserror = "1.0.40" +lazy_static = "1.4.0" +toml = "0.7.3" # Add TOML parsing support ``` -## Implementation Steps +### 2. Replace the name_fix Function -### 1. Project Setup and Dependencies - -1. Update the Cargo.toml files with necessary dependencies: - - redis (for Redis client) - - walkdir (for directory traversal) - - pulldown-cmark (for Markdown to HTML conversion) - - thiserror (for error handling) - - clap (for CLI argument parsing in doctreecmd) - -### 2. Core Library Structure - -1. **Error Module** - - Create a custom error type using thiserror - - Define specific error variants for different failure scenarios - -2. **Storage Module** - - Implement the RedisStorage struct to handle Redis operations - - Provide methods for storing, retrieving, and deleting collection entries - - Implement connection pooling for efficient Redis access - -3. **Utils Module** - - Implement utility functions like name_fix (equivalent to tools.NameFix in Go) - - Implement markdown to HTML conversion using pulldown-cmark - -### 3. Collection Implementation - -1. **Collection Module** - - Implement the Collection struct to represent a collection of documents - - Implement the CollectionBuilder for creating Collection instances - - Implement methods for scanning directories, managing pages and files - -2. **Collection Builder Pattern** - - Create a builder pattern for Collection creation - - Allow configuration of Collection properties before building - -### 4. DocTree Implementation - -1. **DocTree Module** - - Implement the DocTree struct to manage multiple collections - - Implement the DocTreeBuilder for creating DocTree instances - - Implement methods for managing collections and accessing documents - -2. **DocTree Builder Pattern** - - Create a builder pattern for DocTree creation - - Allow adding collections and setting default collection before building - -### 5. Include Processor Implementation - -1. **Include Module** - - Implement the IncludeProcessor to handle include directives - - Implement parsing of include directives - - Implement recursive processing of includes - -### 6. CLI Example - -1. **Update doctreecmd** - - Implement a minimal CLI interface using clap - - Provide commands for basic operations: - - Scanning a directory - - Listing collections - - Getting page content - - Getting HTML content - -## Detailed Module Breakdown - -### Error Module (src/error.rs) +Replace the current `name_fix` function in `utils.rs` with the one from the sal library: ```rust -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum DocTreeError { - #[error("IO error: {0}")] - IoError(#[from] std::io::Error), +pub fn name_fix(text: &str) -> String { + let mut result = String::with_capacity(text.len()); - #[error("Redis error: {0}")] - RedisError(#[from] redis::RedisError), + let mut last_was_underscore = false; + for c in text.chars() { + // Keep only ASCII characters + if c.is_ascii() { + // Replace specific characters with underscore + if c.is_whitespace() || c == ',' || c == '-' || c == '"' || c == '\'' || + c == '#' || c == '!' || c == '(' || c == ')' || c == '[' || c == ']' || + c == '=' || c == '+' || c == '<' || c == '>' || c == '@' || c == '$' || + c == '%' || c == '^' || c == '&' || c == '*' { + // Only add underscore if the last character wasn't an underscore + if !last_was_underscore { + result.push('_'); + last_was_underscore = true; + } + } else { + // Add the character as is (will be converted to lowercase later) + result.push(c); + last_was_underscore = false; + } + } + // Non-ASCII characters are simply skipped + } - #[error("Collection not found: {0}")] - CollectionNotFound(String), - - #[error("Page not found: {0}")] - PageNotFound(String), - - #[error("File not found: {0}")] - FileNotFound(String), - - #[error("Invalid include directive: {0}")] - InvalidIncludeDirective(String), - - #[error("No default collection set")] - NoDefaultCollection, - - #[error("Invalid number of arguments")] - InvalidArgumentCount, + // Convert to lowercase + return result.to_lowercase(); } - -pub type Result = std::result::Result; ``` -### Storage Module (src/storage.rs) +### 3. Add Collection Configuration Struct + +Create a new struct to represent the configuration found in `.collection` files: ```rust -use redis::{Client, Commands, Connection}; -use crate::error::{DocTreeError, Result}; - -pub struct RedisStorage { - client: Client, +#[derive(Deserialize, Default)] +struct CollectionConfig { + name: Option, + // Add other configuration options as needed } +``` -impl RedisStorage { - pub fn new(url: &str) -> Result { - let client = Client::open(url)?; - Ok(Self { client }) - } - - pub fn get_connection(&self) -> Result { - Ok(self.client.get_connection()?) - } - - pub fn store_collection_entry(&self, collection: &str, key: &str, value: &str) -> Result<()> { - let mut conn = self.get_connection()?; - let collection_key = format!("collections:{}", collection); - conn.hset(collection_key, key, value)?; +### 4. Add Scan Collections Method to DocTree + +Add a new method to the DocTree struct to recursively scan directories for `.collection` files: + +```rust +impl DocTree { + /// Recursively scan directories for .collection files and add them as collections + /// + /// # Arguments + /// + /// * `root_path` - The root path to start scanning from + /// + /// # Returns + /// + /// Ok(()) on success or an error + pub fn scan_collections>(&mut self, root_path: P) -> Result<()> { + let root_path = root_path.as_ref(); + + // Walk through the directory tree + for entry in WalkDir::new(root_path).follow_links(true) { + let entry = match entry { + Ok(entry) => entry, + Err(e) => { + eprintln!("Error walking directory: {}", e); + continue; + } + }; + + // Skip non-directories + if !entry.file_type().is_dir() { + continue; + } + + // Check if this directory contains a .collection file + let collection_file_path = entry.path().join(".collection"); + if collection_file_path.exists() { + // Found a collection directory + let dir_path = entry.path(); + + // Get the directory name as a fallback collection name + let dir_name = dir_path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or("unnamed"); + + // Try to read and parse the .collection file + let collection_name = match fs::read_to_string(&collection_file_path) { + Ok(content) => { + // Parse as TOML + match toml::from_str::(&content) { + Ok(config) => { + // Use the name from config if available, otherwise use directory name + config.name.unwrap_or_else(|| dir_name.to_string()) + }, + Err(e) => { + eprintln!("Error parsing .collection file at {:?}: {}", collection_file_path, e); + dir_name.to_string() + } + } + }, + Err(e) => { + eprintln!("Error reading .collection file at {:?}: {}", collection_file_path, e); + dir_name.to_string() + } + }; + + // Add the collection to the DocTree + match self.add_collection(dir_path, &collection_name) { + Ok(_) => { + println!("Added collection '{}' from {:?}", collection_name, dir_path); + }, + Err(e) => { + eprintln!("Error adding collection '{}' from {:?}: {}", collection_name, dir_path, e); + } + } + } + } + Ok(()) } - - pub fn get_collection_entry(&self, collection: &str, key: &str) -> Result { - let mut conn = self.get_connection()?; - let collection_key = format!("collections:{}", collection); - let value: String = conn.hget(collection_key, key)?; - Ok(value) - } - - // Additional methods for Redis operations } ``` -### Utils Module (src/utils.rs) +### 5. Update the DocTreeBuilder + +Update the DocTreeBuilder to include a method for scanning collections: ```rust -use pulldown_cmark::{Parser, Options, html}; - -pub fn name_fix(name: &str) -> String { - // Implementation of name_fix similar to tools.NameFix in Go - // Normalize the name by converting to lowercase, replacing spaces with hyphens, etc. -} - -pub fn markdown_to_html(markdown: &str) -> String { - let mut options = Options::empty(); - options.insert(Options::ENABLE_TABLES); - options.insert(Options::ENABLE_FOOTNOTES); - options.insert(Options::ENABLE_STRIKETHROUGH); - - let parser = Parser::new_ext(markdown, options); - let mut html_output = String::new(); - html::push_html(&mut html_output, parser); - - html_output -} -``` - -### Collection Module (src/collection.rs) - -```rust -use std::path::{Path, PathBuf}; -use walkdir::WalkDir; -use crate::error::Result; -use crate::storage::RedisStorage; -use crate::utils::name_fix; - -pub struct Collection { - pub path: PathBuf, - pub name: String, - storage: RedisStorage, -} - -pub struct CollectionBuilder { - path: PathBuf, - name: String, - storage: Option, -} - -impl Collection { - pub fn builder>(path: P, name: &str) -> CollectionBuilder { - CollectionBuilder { - path: path.as_ref().to_path_buf(), - name: name_fix(name), - storage: None, - } - } - - pub fn scan(&self) -> Result<()> { - // Implementation of scanning directory and storing in Redis - } - - pub fn page_get(&self, page_name: &str) -> Result { - // Implementation of getting page content - } - - // Additional methods for Collection -} - -impl CollectionBuilder { - pub fn with_storage(mut self, storage: RedisStorage) -> Self { - self.storage = Some(storage); - self - } - - pub fn build(self) -> Result { - let storage = self.storage.ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::Other, "Storage not provided") +impl DocTreeBuilder { + /// Scan for collections in the given root path + /// + /// # Arguments + /// + /// * `root_path` - The root path to scan for collections + /// + /// # Returns + /// + /// Self for method chaining or an error + pub fn scan_collections>(self, root_path: P) -> Result { + // Ensure storage is set + let storage = self.storage.as_ref().ok_or_else(|| { + DocTreeError::MissingParameter("storage".to_string()) })?; - let collection = Collection { - path: self.path, - name: self.name, - storage, - }; - - Ok(collection) - } -} -``` - -### DocTree Module (src/doctree.rs) - -```rust -use std::collections::HashMap; -use std::path::Path; -use crate::collection::{Collection, CollectionBuilder}; -use crate::error::{DocTreeError, Result}; -use crate::storage::RedisStorage; - -pub struct DocTree { - collections: HashMap, - default_collection: Option, - storage: RedisStorage, -} - -pub struct DocTreeBuilder { - collections: HashMap, - default_collection: Option, - storage: Option, -} - -impl DocTree { - pub fn builder() -> DocTreeBuilder { - DocTreeBuilder { + // Create a temporary DocTree to scan collections + let mut temp_doctree = DocTree { collections: HashMap::new(), default_collection: None, - storage: None, - } - } - - pub fn add_collection>(&mut self, path: P, name: &str) -> Result<&Collection> { - // Implementation of adding a collection - } - - // Additional methods for DocTree -} - -impl DocTreeBuilder { - pub fn with_storage(mut self, storage: RedisStorage) -> Self { - self.storage = Some(storage); - self - } - - pub fn with_collection>(mut self, path: P, name: &str) -> Result { - // Implementation of adding a collection during building - Ok(self) - } - - pub fn with_default_collection(mut self, name: &str) -> Self { - self.default_collection = Some(name.to_string()); - self - } - - pub fn build(self) -> Result { - let storage = self.storage.ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::Other, "Storage not provided") - })?; - - let doctree = DocTree { - collections: self.collections, - default_collection: self.default_collection, - storage, + storage: storage.clone(), + name: self.name.clone().unwrap_or_default(), + path: self.path.clone().unwrap_or_else(|| PathBuf::from("")), }; - Ok(doctree) + // Scan for collections + temp_doctree.scan_collections(root_path)?; + + // Create a new builder with the scanned collections + let mut new_builder = self; + for (name, collection) in temp_doctree.collections { + new_builder.collections.insert(name, collection); + } + + Ok(new_builder) } } ``` -### Include Module (src/include.rs) +### 6. Add a Convenience Function to the Library + +Add a convenience function to the library for creating a DocTree by scanning a directory: ```rust -use crate::doctree::DocTree; -use crate::error::Result; - -pub fn process_includes(content: &str, collection_name: &str, doctree: &DocTree) -> Result { - // Implementation of processing include directives -} - -fn parse_include_line(line: &str) -> Result<(Option, Option)> { - // Implementation of parsing include directives -} - -fn handle_include(page_name: &str, collection_name: &str, doctree: &DocTree) -> Result { - // Implementation of handling include directives -} -``` - -### Main Library File (src/lib.rs) - -```rust -mod error; -mod storage; -mod utils; -mod collection; -mod doctree; -mod include; - -pub use error::{DocTreeError, Result}; -pub use storage::RedisStorage; -pub use collection::{Collection, CollectionBuilder}; -pub use doctree::{DocTree, DocTreeBuilder}; -pub use include::process_includes; -``` - -### CLI Example (doctreecmd/src/main.rs) - -```rust -use clap::{App, Arg, SubCommand}; -use doctree::{DocTree, RedisStorage}; - -fn main() -> Result<(), Box> { - let matches = App::new("DocTree CLI") - .version("0.1.0") - .author("Your Name") - .about("A tool to manage document collections") - .subcommand( - SubCommand::with_name("scan") - .about("Scan a directory and create a collection") - .arg(Arg::with_name("path").required(true)) - .arg(Arg::with_name("name").required(true)), - ) - .subcommand( - SubCommand::with_name("list") - .about("List collections"), - ) - .subcommand( - SubCommand::with_name("get") - .about("Get page content") - .arg(Arg::with_name("collection").required(true)) - .arg(Arg::with_name("page").required(true)), - ) - .get_matches(); - - // Implementation of CLI commands - - Ok(()) -} -``` - -## Example Usage - -Here's how the library would be used with the builder pattern: - -```rust -use doctree::{DocTree, RedisStorage}; - -fn main() -> Result<(), Box> { - // Create a Redis storage instance +/// Create a new DocTree by scanning a directory for collections +/// +/// # Arguments +/// +/// * `root_path` - The root path to scan for collections +/// +/// # Returns +/// +/// A new DocTree or an error +pub fn from_directory>(root_path: P) -> Result { let storage = RedisStorage::new("redis://localhost:6379")?; - // Create a DocTree instance using the builder pattern - let mut doctree = DocTree::builder() - .with_storage(storage.clone()) - .with_collection("path/to/collection", "my-collection")? - .with_default_collection("my-collection") - .build()?; - - // Get page content - let content = doctree.page_get("my-collection", "page-name")?; - println!("Page content: {}", content); - - // Get HTML content - let html = doctree.page_get_html("my-collection", "page-name")?; - println!("HTML content: {}", html); - - Ok(()) + DocTree::builder() + .with_storage(storage) + .scan_collections(root_path)? + .build() } ``` -## Testing Strategy +## Implementation Flow Diagram -1. **Unit Tests** - - Test individual components in isolation - - Mock Redis for testing storage operations - - Test utility functions +```mermaid +flowchart TD + A[Start] --> B[Update Dependencies] + B --> C[Replace name_fix function] + C --> D[Add CollectionConfig struct] + D --> E[Add scan_collections method to DocTree] + E --> F[Update DocTreeBuilder] + F --> G[Add convenience function] + G --> H[End] +``` -2. **Integration Tests** - - Test the interaction between components - - Test the builder pattern - - Test include processing +## Component Interaction Diagram -3. **End-to-End Tests** - - Test the complete workflow with real files - - Test the CLI interface +```mermaid +graph TD + A[DocTree] -->|manages| B[Collections] + C[scan_collections] -->|finds| D[.collection files] + D -->|parsed as| E[TOML] + E -->|extracts| F[Collection Name] + C -->|creates| B + G[name_fix] -->|processes| F + G -->|processes| H[File Names] + B -->|contains| H +``` -## Timeline +## Testing Plan -1. **Project Setup and Dependencies**: 1 day -2. **Core Library Structure**: 2 days -3. **Collection Implementation**: 2 days -4. **DocTree Implementation**: 2 days -5. **Include Processor Implementation**: 1 day -6. **CLI Example**: 1 day -7. **Testing and Documentation**: 2 days - -Total estimated time: 11 days \ No newline at end of file +1. Create test directories with `.collection` files in various formats +2. Test the scan_collections method with these directories +3. Verify that collections are created correctly with the expected names +4. Verify that all files under the collection directories are included in the collections +5. Test edge cases such as empty `.collection` files, invalid TOML, etc. \ No newline at end of file diff --git a/doctreecmd/src/main.rs b/doctreecmd/src/main.rs index cb2100f..178a464 100644 --- a/doctreecmd/src/main.rs +++ b/doctreecmd/src/main.rs @@ -1,5 +1,5 @@ use clap::{App, Arg, SubCommand}; -use doctree::{DocTree, RedisStorage, Result}; +use doctree::{DocTree, RedisStorage, Result, from_directory}; use std::path::Path; fn main() -> Result<()> { @@ -17,11 +17,41 @@ fn main() -> Result<()> { SubCommand::with_name("list") .about("List collections"), ) + .subcommand( + SubCommand::with_name("scan-collections") + .about("Recursively scan directories for .collection files") + .arg(Arg::with_name("path").required(true).help("Root path to scan for collections")), + ) + .subcommand( + SubCommand::with_name("scan-and-info") + .about("Scan collections and show detailed information") + .arg(Arg::with_name("path").required(true).help("Root path to scan for collections")) + .arg(Arg::with_name("collection").help("Name of the collection (optional)")), + ) + .subcommand( + SubCommand::with_name("info") + .about("Show detailed information about collections") + .arg(Arg::with_name("collection").help("Name of the collection (optional)")), + ) .subcommand( SubCommand::with_name("get") .about("Get page content") - .arg(Arg::with_name("collection").required(true).help("Name of the collection")) - .arg(Arg::with_name("page").required(true).help("Name of the page")), + .arg(Arg::with_name("collection") + .short("c".chars().next().unwrap()) + .long("collection") + .takes_value(true) + .help("Name of the collection (optional)")) + .arg(Arg::with_name("page") + .short("p".chars().next().unwrap()) + .long("page") + .required(true) + .takes_value(true) + .help("Name of the page")) + .arg(Arg::with_name("format") + .short("f".chars().next().unwrap()) + .long("format") + .takes_value(true) + .help("Output format (html or markdown, default: markdown)")), ) .subcommand( SubCommand::with_name("html") @@ -29,6 +59,15 @@ fn main() -> Result<()> { .arg(Arg::with_name("collection").required(true).help("Name of the collection")) .arg(Arg::with_name("page").required(true).help("Name of the page")), ) + .subcommand( + SubCommand::with_name("delete-collection") + .about("Delete a collection from Redis") + .arg(Arg::with_name("collection").required(true).help("Name of the collection")), + ) + .subcommand( + SubCommand::with_name("reset") + .about("Delete all collections from Redis"), + ) .get_matches(); // Create a Redis storage instance @@ -59,17 +98,257 @@ fn main() -> Result<()> { } } } else if let Some(matches) = matches.subcommand_matches("get") { - let collection = matches.value_of("collection").unwrap(); + let collection = matches.value_of("collection"); let page = matches.value_of("page").unwrap(); + let format = matches.value_of("format").unwrap_or("markdown"); - let content = doctree.page_get(Some(collection), page)?; - println!("{}", content); + if format.to_lowercase() == "html" { + let html = doctree.page_get_html(collection, page)?; + println!("{}", html); + } else { + let content = doctree.page_get(collection, page)?; + println!("{}", content); + } } else if let Some(matches) = matches.subcommand_matches("html") { let collection = matches.value_of("collection").unwrap(); let page = matches.value_of("page").unwrap(); let html = doctree.page_get_html(Some(collection), page)?; println!("{}", html); + } else if let Some(matches) = matches.subcommand_matches("delete-collection") { + let collection = matches.value_of("collection").unwrap(); + + println!("Deleting collection '{}' from Redis...", collection); + doctree.delete_collection(collection)?; + println!("Collection '{}' deleted successfully", collection); + } else if let Some(_) = matches.subcommand_matches("reset") { + println!("Deleting all collections from Redis..."); + doctree.delete_all_collections()?; + println!("All collections deleted successfully"); + } else if let Some(matches) = matches.subcommand_matches("scan-collections") { + let path = matches.value_of("path").unwrap(); + + println!("Recursively scanning for collections in: {}", path); + + // Use the from_directory function to create a DocTree with all collections + let doctree = from_directory(Path::new(path))?; + + // Print the discovered collections + let collections = doctree.list_collections(); + if collections.is_empty() { + println!("No collections found"); + } else { + println!("Discovered collections:"); + for collection in collections { + println!("- {}", collection); + } + } + } else if let Some(matches) = matches.subcommand_matches("scan-and-info") { + let path = matches.value_of("path").unwrap(); + let collection_name = matches.value_of("collection"); + + println!("Recursively scanning for collections in: {}", path); + + // Use the from_directory function to create a DocTree with all collections + let doctree = from_directory(Path::new(path))?; + + // Print the discovered collections + let collections = doctree.list_collections(); + if collections.is_empty() { + println!("No collections found"); + return Ok(()); + } + + println!("Discovered collections:"); + for collection in &collections { + println!("- {}", collection); + } + + println!("\nDetailed Collection Information:"); + + if let Some(name) = collection_name { + // Show info for a specific collection + match doctree.get_collection(name) { + Ok(collection) => { + println!("Collection Information for '{}':", name); + println!(" Path: {:?}", collection.path); + println!(" Redis Key: collections:{}", collection.name); + + // List documents + match collection.page_list() { + Ok(pages) => { + println!(" Documents ({}):", pages.len()); + for page in pages { + match collection.page_get_path(&page) { + Ok(path) => { + println!(" - {} => Redis: collections:{} / {}", path, collection.name, page); + }, + Err(_) => { + println!(" - {}", page); + } + } + } + }, + Err(e) => println!(" Error listing documents: {}", e), + } + + // List files + match collection.file_list() { + Ok(files) => { + // Filter images + let images: Vec = files.iter() + .filter(|f| + f.ends_with(".png") || f.ends_with(".jpg") || + f.ends_with(".jpeg") || f.ends_with(".gif") || + f.ends_with(".svg")) + .cloned() + .collect(); + + println!(" Images ({}):", images.len()); + for image in images { + println!(" - {} => Redis: collections:{} / {}", image, collection.name, image); + } + + // Filter other files + let other_files: Vec = files.iter() + .filter(|f| + !f.ends_with(".png") && !f.ends_with(".jpg") && + !f.ends_with(".jpeg") && !f.ends_with(".gif") && + !f.ends_with(".svg")) + .cloned() + .collect(); + + println!(" Other Files ({}):", other_files.len()); + for file in other_files { + println!(" - {} => Redis: collections:{} / {}", file, collection.name, file); + } + }, + Err(e) => println!(" Error listing files: {}", e), + } + }, + Err(e) => println!("Error: {}", e), + } + } else { + // Show info for all collections + for name in collections { + if let Ok(collection) = doctree.get_collection(&name) { + println!("- {} (Redis Key: collections:{})", name, collection.name); + println!(" Path: {:?}", collection.path); + + // Count documents and images + if let Ok(pages) = collection.page_list() { + println!(" Documents: {}", pages.len()); + } + + if let Ok(files) = collection.file_list() { + let image_count = files.iter() + .filter(|f| + f.ends_with(".png") || f.ends_with(".jpg") || + f.ends_with(".jpeg") || f.ends_with(".gif") || + f.ends_with(".svg")) + .count(); + println!(" Images: {}", image_count); + println!(" Other Files: {}", files.len() - image_count); + } + } + } + } + } else if let Some(matches) = matches.subcommand_matches("info") { + let collection_name = matches.value_of("collection"); + + if let Some(name) = collection_name { + // Show info for a specific collection + match doctree.get_collection(name) { + Ok(collection) => { + println!("Collection Information for '{}':", name); + println!(" Path: {:?}", collection.path); + println!(" Redis Key: collections:{}", collection.name); + + // List documents + match collection.page_list() { + Ok(pages) => { + println!(" Documents ({}):", pages.len()); + for page in pages { + match collection.page_get_path(&page) { + Ok(path) => { + println!(" - {} => Redis: collections:{} / {}", path, collection.name, page); + }, + Err(_) => { + println!(" - {}", page); + } + } + } + }, + Err(e) => println!(" Error listing documents: {}", e), + } + + // List files + match collection.file_list() { + Ok(files) => { + // Filter images + let images: Vec = files.iter() + .filter(|f| + f.ends_with(".png") || f.ends_with(".jpg") || + f.ends_with(".jpeg") || f.ends_with(".gif") || + f.ends_with(".svg")) + .cloned() + .collect(); + + println!(" Images ({}):", images.len()); + for image in images { + println!(" - {} => Redis: collections:{} / {}", image, collection.name, image); + } + + // Filter other files + let other_files: Vec = files.iter() + .filter(|f| + !f.ends_with(".png") && !f.ends_with(".jpg") && + !f.ends_with(".jpeg") && !f.ends_with(".gif") && + !f.ends_with(".svg")) + .cloned() + .collect(); + + println!(" Other Files ({}):", other_files.len()); + for file in other_files { + println!(" - {} => Redis: collections:{} / {}", file, collection.name, file); + } + }, + Err(e) => println!(" Error listing files: {}", e), + } + }, + Err(e) => println!("Error: {}", e), + } + } else { + // Show info for all collections + let collections = doctree.list_collections(); + if collections.is_empty() { + println!("No collections found"); + } else { + println!("Collections:"); + for name in collections { + if let Ok(collection) = doctree.get_collection(&name) { + println!("- {} (Redis Key: collections:{})", name, collection.name); + println!(" Path: {:?}", collection.path); + + // Count documents and images + if let Ok(pages) = collection.page_list() { + println!(" Documents: {}", pages.len()); + } + + if let Ok(files) = collection.file_list() { + let image_count = files.iter() + .filter(|f| + f.ends_with(".png") || f.ends_with(".jpg") || + f.ends_with(".jpeg") || f.ends_with(".gif") || + f.ends_with(".svg")) + .count(); + println!(" Images: {}", image_count); + println!(" Other Files: {}", files.len() - image_count); + } + } + } + } + } } else { println!("No command specified. Use --help for usage information."); } diff --git a/example_commands.sh b/example_commands.sh new file mode 100755 index 0000000..49a69a6 --- /dev/null +++ b/example_commands.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Change to the directory where the script is located +cd "$(dirname "$0")" +# Exit immediately if a command exits with a non-zero status +set -e + +cd doctreecmd + +echo "=== Scanning Collections ===" +cargo run -- scan-collections ../examples + +echo -e "\n=== Listing Collections ===" +cargo run -- list + +echo -e "\n=== Getting Document (Markdown) ===" +cargo run -- get -c grid_documentation -p introduction.md + +echo -e "\n=== Getting Document (HTML) ===" +cargo run -- get -c grid_documentation -p introduction.md -f html + +echo -e "\n=== Deleting Collection ===" +cargo run -- delete-collection grid_documentation + +echo -e "\n=== Listing Remaining Collections ===" +cargo run -- list + +echo -e "\n=== Resetting All Collections ===" +cargo run -- reset + +echo -e "\n=== Verifying Reset ===" +cargo run -- list \ No newline at end of file diff --git a/examples/grid1/.collection b/examples/grid1/.collection index e69de29..7d30e20 100644 --- a/examples/grid1/.collection +++ b/examples/grid1/.collection @@ -0,0 +1 @@ +name = "Grid Documentation" diff --git a/examples/parent/docs2/.collection b/examples/parent/docs2/.collection index e69de29..e014ab3 100644 --- a/examples/parent/docs2/.collection +++ b/examples/parent/docs2/.collection @@ -0,0 +1 @@ +name = "supercollection" \ No newline at end of file diff --git a/runexample.sh b/runexample.sh new file mode 100755 index 0000000..93e5bd3 --- /dev/null +++ b/runexample.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Change to the directory where the script is located +cd "$(dirname "$0")" +# Exit immediately if a command exits with a non-zero status +set -e + +cd doctreecmd + +# First, scan the collections +echo "=== Scanning Collections ===" +cargo run -- scan-and-info ../examples supercollection + +# Get a document in markdown format +echo -e "\n=== Getting Document (Markdown) ===" +cargo run -- get -c supercollection -p 01_features.md + +# Get a document in HTML format +echo -e "\n=== Getting Document (HTML) ===" +cargo run -- get -c supercollection -p 01_features.md -f html + +# Get a document without specifying collection +echo -e "\n=== Getting Document (Default Collection) ===" +cargo run -- get -p 01_features.md + +# Delete a specific collection +echo -e "\n=== Deleting Collection ===" +cargo run -- delete-collection grid_documentation + +# List remaining collections +echo -e "\n=== Listing Remaining Collections ===" +cargo run -- list + +# Reset all collections +echo -e "\n=== Resetting All Collections ===" +cargo run -- reset + +# Verify all collections are gone +echo -e "\n=== Verifying Reset ===" +cargo run -- list \ No newline at end of file