diff --git a/doctree/src/collection.rs b/doctree/src/collection.rs index 8dcf7b5..8efc217 100644 --- a/doctree/src/collection.rs +++ b/doctree/src/collection.rs @@ -15,10 +15,10 @@ use ipfs_api::{IpfsApi, IpfsClient}; pub struct Collection { /// Base path of the collection pub path: PathBuf, - + /// Name of the collection (namefixed) pub name: String, - + /// Redis storage backend pub storage: RedisStorage, } @@ -27,10 +27,10 @@ pub struct Collection { pub struct CollectionBuilder { /// Base path of the collection path: PathBuf, - + /// Name of the collection (namefixed) name: String, - + /// Redis storage backend storage: Option, } @@ -53,7 +53,7 @@ impl Collection { storage: None, } } - + /// Scan walks over the path and finds all files and .md files /// It stores the relative positions in Redis /// @@ -62,7 +62,7 @@ impl Collection { /// Ok(()) on success or an error pub fn scan(&self) -> Result<()> { println!("DEBUG: Scanning collection '{}' at path {:?}", self.name, self.path); - + // Delete existing collection data if any println!("DEBUG: Deleting existing collection data from Redis key 'collections:{}'", self.name); self.storage.delete_collection(&self.name)?; @@ -71,11 +71,11 @@ impl Collection { .unwrap_or_else(|_| self.path.clone()) .to_string_lossy() .to_string(); - + println!("DEBUG: Storing collection path in Redis key 'collections:{}:path'", self.name); self.storage.store_collection_path(&self.name, &absolute_path)?; self.storage.store_collection_path(&self.name, &self.path.to_string_lossy())?; - + // Walk through the directory let walker = WalkDir::new(&self.path); for entry_result in walker { @@ -88,18 +88,18 @@ impl Collection { continue; } }; - + // Skip directories if entry.file_type().is_dir() { continue; } - + // Skip files that start with a dot (.) let file_name = entry.file_name().to_string_lossy(); if file_name.starts_with(".") { continue; } - + // Get the relative path from the base path let rel_path = match entry.path().strip_prefix(&self.path) { Ok(path) => path, @@ -109,11 +109,11 @@ impl Collection { continue; } }; - + // Get the filename and apply namefix let filename = entry.file_name().to_string_lossy().to_string(); let namefixed_filename = name_fix(&filename); - + // Determine if this is a document (markdown file) or an image let is_markdown = filename.to_lowercase().ends_with(".md"); let is_image = filename.to_lowercase().ends_with(".png") || @@ -121,7 +121,7 @@ impl Collection { filename.to_lowercase().ends_with(".jpeg") || filename.to_lowercase().ends_with(".gif") || filename.to_lowercase().ends_with(".svg"); - + let file_type = if is_markdown { "document" } else if is_image { @@ -129,22 +129,22 @@ impl Collection { } else { "file" }; - + // Store in Redis using the namefixed filename as the key // Store the original relative path to preserve case and special characters println!("DEBUG: Storing {} '{}' in Redis key 'collections:{}' with key '{}' and value '{}'", file_type, filename, self.name, namefixed_filename, rel_path.to_string_lossy()); - + self.storage.store_collection_entry( &self.name, &namefixed_filename, &rel_path.to_string_lossy() )?; } - + Ok(()) } - + /// Get a page by name and return its markdown content /// /// # Arguments @@ -157,14 +157,14 @@ impl Collection { pub fn page_get(&self, page_name: &str) -> Result { // Apply namefix to the page name let namefixed_page_name = name_fix(page_name); - + // Ensure it has .md extension let namefixed_page_name = ensure_md_extension(&namefixed_page_name); - + // Get the relative path from Redis let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_page_name) .map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?; - + // Check if the path is valid if self.path.as_os_str().is_empty() { // If the path is empty, we're working with a collection loaded from Redis @@ -174,18 +174,18 @@ impl Collection { format!("File path not available for {} in collection {}", page_name, self.name) ))); } - + // Read the file let full_path = self.path.join(rel_path); let content = fs::read_to_string(full_path) .map_err(|e| DocTreeError::IoError(e))?; - + // Skip include processing at this level to avoid infinite recursion // Include processing will be done at the higher level - + Ok(content) } - + /// Create or update a page in the collection /// /// # Arguments @@ -199,27 +199,27 @@ impl Collection { pub fn page_set(&self, page_name: &str, content: &str) -> Result<()> { // Apply namefix to the page name let namefixed_page_name = name_fix(page_name); - + // Ensure it has .md extension let namefixed_page_name = ensure_md_extension(&namefixed_page_name); - + // Create the full path let full_path = self.path.join(&namefixed_page_name); - + // Create directories if needed if let Some(parent) = full_path.parent() { fs::create_dir_all(parent).map_err(DocTreeError::IoError)?; } - + // Write content to file fs::write(&full_path, content).map_err(DocTreeError::IoError)?; - + // Update Redis self.storage.store_collection_entry(&self.name, &namefixed_page_name, &namefixed_page_name)?; - + Ok(()) } - + /// Delete a page from the collection /// /// # Arguments @@ -232,24 +232,24 @@ impl Collection { pub fn page_delete(&self, page_name: &str) -> Result<()> { // Apply namefix to the page name let namefixed_page_name = name_fix(page_name); - + // Ensure it has .md extension let namefixed_page_name = ensure_md_extension(&namefixed_page_name); - + // Get the relative path from Redis let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_page_name) .map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?; - + // Delete the file let full_path = self.path.join(rel_path); fs::remove_file(full_path).map_err(DocTreeError::IoError)?; - + // Remove from Redis self.storage.delete_collection_entry(&self.name, &namefixed_page_name)?; - + Ok(()) } - + /// List all pages in the collection /// /// # Returns @@ -258,15 +258,15 @@ impl Collection { pub fn page_list(&self) -> Result> { // Get all keys from Redis let keys = self.storage.list_collection_entries(&self.name)?; - + // Filter to only include .md files let pages = keys.into_iter() .filter(|key| key.ends_with(".md")) .collect(); - + Ok(pages) } - + /// Get the URL for a file /// /// # Arguments @@ -279,17 +279,17 @@ impl Collection { pub fn file_get_url(&self, file_name: &str) -> Result { // Apply namefix to the file name let namefixed_file_name = name_fix(file_name); - + // Get the relative path from Redis let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_file_name) .map_err(|_| DocTreeError::FileNotFound(file_name.to_string()))?; - + // Construct a URL for the file let url = format!("/collections/{}/files/{}", self.name, rel_path); - + Ok(url) } - + /// Add or update a file in the collection /// /// # Arguments @@ -303,24 +303,24 @@ impl Collection { pub fn file_set(&self, file_name: &str, content: &[u8]) -> Result<()> { // Apply namefix to the file name let namefixed_file_name = name_fix(file_name); - + // Create the full path let full_path = self.path.join(&namefixed_file_name); - + // Create directories if needed if let Some(parent) = full_path.parent() { fs::create_dir_all(parent).map_err(DocTreeError::IoError)?; } - + // Write content to file fs::write(&full_path, content).map_err(DocTreeError::IoError)?; - + // Update Redis self.storage.store_collection_entry(&self.name, &namefixed_file_name, &namefixed_file_name)?; - + Ok(()) } - + /// Delete a file from the collection /// /// # Arguments @@ -333,21 +333,21 @@ impl Collection { pub fn file_delete(&self, file_name: &str) -> Result<()> { // Apply namefix to the file name let namefixed_file_name = name_fix(file_name); - + // Get the relative path from Redis let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_file_name) .map_err(|_| DocTreeError::FileNotFound(file_name.to_string()))?; - + // Delete the file let full_path = self.path.join(rel_path); fs::remove_file(full_path).map_err(DocTreeError::IoError)?; - + // Remove from Redis self.storage.delete_collection_entry(&self.name, &namefixed_file_name)?; - + Ok(()) } - + /// List all files (non-markdown) in the collection /// /// # Returns @@ -356,15 +356,15 @@ impl Collection { pub fn file_list(&self) -> Result> { // Get all keys from Redis let keys = self.storage.list_collection_entries(&self.name)?; - + // Filter to exclude .md files let files = keys.into_iter() .filter(|key| !key.ends_with(".md")) .collect(); - + Ok(files) } - + /// Get the relative path of a page in the collection /// /// # Arguments @@ -377,15 +377,15 @@ impl Collection { pub fn page_get_path(&self, page_name: &str) -> Result { // Apply namefix to the page name let namefixed_page_name = name_fix(page_name); - + // Ensure it has .md extension let namefixed_page_name = ensure_md_extension(&namefixed_page_name); - + // Get the relative path from Redis self.storage.get_collection_entry(&self.name, &namefixed_page_name) .map_err(|_| DocTreeError::PageNotFound(page_name.to_string())) } - + /// Get a page by name and return its HTML content /// /// # Arguments @@ -399,20 +399,20 @@ impl Collection { pub fn page_get_html(&self, page_name: &str, doctree: Option<&crate::doctree::DocTree>) -> Result { // Get the markdown content let markdown = self.page_get(page_name)?; - + // Process includes if doctree is provided let processed_markdown = if let Some(dt) = doctree { process_includes(&markdown, &self.name, dt)? } else { markdown }; - + // Convert markdown to HTML let html = markdown_to_html(&processed_markdown); - + Ok(html) } - + /// Get information about the Collection /// /// # Returns @@ -424,9 +424,8 @@ impl Collection { info.insert("path".to_string(), self.path.to_string_lossy().to_string()); info } -} - /// Exports files and images from the collection to IPFS, encrypts them, and generates a CSV manifest. + /// Exports files and images from the collection to IPFS synchronously, encrypting them, and generating a CSV manifest. /// /// # Arguments /// @@ -435,8 +434,16 @@ impl Collection { /// # Returns /// /// Ok(()) on success or an error. -impl Collection { - /// Exports files and images from the collection to IPFS, encrypts them, and generates a CSV manifest. + pub fn export_to_ipfs(&self, output_csv_path: &Path) -> Result<()> { + // Create a new tokio runtime and block on the async export function + tokio::runtime::Runtime::new()?.block_on(async { + self.export_to_ipfs_async(output_csv_path).await + })?; + + Ok(()) + } + + /// Exports files and images from the collection to IPFS asynchronously, encrypts them, and generates a CSV manifest. /// /// # Arguments /// @@ -445,7 +452,7 @@ impl Collection { /// # Returns /// /// Ok(()) on success or an error. - pub async fn export_to_ipfs(&self, output_csv_path: &Path) -> Result<()> { + pub async fn export_to_ipfs_async(&self, output_csv_path: &Path) -> Result<()> { use blake3::Hasher; // use chacha20poly1305::{ChaCha20Poly1305, Aead}; use ipfs_api::IpfsClient; @@ -570,7 +577,7 @@ impl CollectionBuilder { self.storage = Some(storage); self } - + /// Build the Collection /// /// # Returns @@ -580,13 +587,13 @@ impl CollectionBuilder { let storage = self.storage.ok_or_else(|| { DocTreeError::MissingParameter("storage".to_string()) })?; - + let collection = Collection { path: self.path, name: self.name, storage, }; - + Ok(collection) } } \ No newline at end of file diff --git a/doctree/src/doctree.rs b/doctree/src/doctree.rs index 9d79da0..34243a0 100644 --- a/doctree/src/doctree.rs +++ b/doctree/src/doctree.rs @@ -1,3 +1,4 @@ + use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; @@ -551,7 +552,7 @@ impl DocTree { for (name, collection) in &self.collections { let csv_file_path = output_dir.join(format!("{}.csv", name)); println!("DEBUG: Exporting collection '{}' to IPFS and generating CSV at {:?}", name, csv_file_path); - if let Err(e) = collection.export_to_ipfs(&csv_file_path).await { + if let Err(e) = collection.export_to_ipfs(&csv_file_path) { eprintln!("Error exporting collection '{}': {}", name, e); // Continue with the next collection } @@ -559,6 +560,26 @@ impl DocTree { Ok(()) } + + /// Exports a specific collection to IPFS synchronously, encrypting its files and generating a CSV manifest. + /// + /// # Arguments + /// + /// * `collection_name` - The name of the collection to export. + /// * `output_csv_path` - The path to save the output CSV file. + /// + /// # Returns + /// + /// Ok(()) on success or an error. + pub fn export_collection_to_ipfs(&self, collection_name: &str, output_csv_path: &Path) -> Result<()> { + // Get the collection + let collection = self.get_collection(collection_name)?; + + // Create a new tokio runtime and block on the async export function + collection.export_to_ipfs(output_csv_path); + + Ok(()) + } } impl DocTreeBuilder { diff --git a/doctreecmd/src/main.rs b/doctreecmd/src/main.rs index b4b1c78..5415176 100644 --- a/doctreecmd/src/main.rs +++ b/doctreecmd/src/main.rs @@ -69,6 +69,13 @@ fn main() -> Result<()> { .about("Delete all collections from Redis") .arg(Arg::with_name("doctree").long("doctree").takes_value(true).help("Name of the doctree (default: 'default')")), ) + .subcommand( + SubCommand::with_name("export_to_ipfs") + .about("Export a collection to IPFS") + .arg(Arg::with_name("collection").required(true).help("Name of the collection")) + .arg(Arg::with_name("output").required(true).help("Output directory for IPFS export")) + .arg(Arg::with_name("doctree").long("doctree").takes_value(true).help("Name of the doctree (default: 'default')")), + ) .get_matches(); // Check if debug mode is enabled @@ -342,6 +349,43 @@ fn main() -> Result<()> { println!("Deleting collection '{}' from Redis in doctree '{}'...", collection, doctree_name); doctree.delete_collection(collection)?; println!("Collection '{}' deleted successfully", collection); + } else if let Some(matches) = matches.subcommand_matches("export_to_ipfs") { + let collection_name = matches.value_of("collection").unwrap(); + let output_path = matches.value_of("output").unwrap(); + let doctree_name = matches.value_of("doctree").unwrap_or("default"); + + if debug_mode { + println!("DEBUG: Exporting collection '{}' from doctree '{}' to IPFS output path '{}'", + collection_name, doctree_name, output_path); + } + + // Create a storage with the specified doctree name + let storage = RedisStorage::new("redis://localhost:6379")?; + storage.set_doctree_name(doctree_name); + storage.set_debug(debug_mode); + + if debug_mode { + println!("DEBUG: Connected to Redis storage"); + } + + // Create a DocTree with the specified doctree name + let mut doctree = DocTree::builder() + .with_storage(storage) + .with_doctree_name(doctree_name) + .build()?; + + // Load collections from Redis + doctree.load_collections_from_redis()?; + + // Get the collection + let collection = doctree.get_collection(collection_name)?; + + // Call the synchronous export_collection_to_ipfs_sync function from the doctree crate + let output_path = Path::new(output_path); + doctree.export_collection_to_ipfs(collection_name, output_path)?; + + println!("Successfully exported collection '{}' to IPFS and generated metadata CSV at {:?}.", collection_name, output_path.join(format!("{}.csv", collection_name))); + } else if let Some(matches) = matches.subcommand_matches("reset") { let doctree_name = matches.value_of("doctree").unwrap_or("default"); @@ -370,6 +414,6 @@ fn main() -> Result<()> { } else { println!("No command specified. Use --help for usage information."); } - + Ok(()) }