herocode/doctree
12
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: herocode:development_mahmoud
Branches Tags
herocode:main
herocode:development_mahmoud
..
compare: herocode:main
Branches Tags
herocode:development_mahmoud
herocode:main

No commits in common. "development_mahmoud" and "main" have entirely different histories.
developmen ... main

20 changed files with 100 additions and 2768 deletions
Show Stats Expand all files Collapse all files

4
.gitignore vendored
View File

@ -62,6 +62,4 @@ docusaurus.config.ts
sidebars.ts sidebars.ts
tsconfig.json tsconfig.json
sccache.log sccache.log
*webmeta.json
.vscode

170
doctree/src/collection.rs
View File

@ -1,11 +1,12 @@
use std::fs;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use walkdir::WalkDir; use walkdir::WalkDir;
use std::fs;
use crate::error::{DocTreeError, Result}; use crate::error::{DocTreeError, Result};
use crate::include::process_includes;
use crate::storage::RedisStorage; use crate::storage::RedisStorage;
use crate::utils::{ensure_md_extension, markdown_to_html, name_fix}; use crate::utils::{name_fix, markdown_to_html, ensure_md_extension};
use crate::include::process_includes;
use rand::Rng;
use ipfs_api::{IpfsApi, IpfsClient}; use ipfs_api::{IpfsApi, IpfsClient};
// use chacha20poly1305::aead::NewAead; // use chacha20poly1305::aead::NewAead;
@ -60,16 +61,10 @@ impl Collection {
/// ///
/// Ok(()) on success or an error /// Ok(()) on success or an error
pub fn scan(&self) -> Result<()> { pub fn scan(&self) -> Result<()> {
println!( println!("DEBUG: Scanning collection '{}' at path {:?}", self.name, self.path);
"DEBUG: Scanning collection '{}' at path {:?}",
self.name, self.path
);
// Delete existing collection data if any // Delete existing collection data if any
println!( println!("DEBUG: Deleting existing collection data from Redis key 'collections:{}'", self.name);
"DEBUG: Deleting existing collection data from Redis key 'collections:{}'",
self.name
);
self.storage.delete_collection(&self.name)?; self.storage.delete_collection(&self.name)?;
// Store the collection's full absolute path in Redis // Store the collection's full absolute path in Redis
let absolute_path = std::fs::canonicalize(&self.path) let absolute_path = std::fs::canonicalize(&self.path)
@ -77,14 +72,9 @@ impl Collection {
.to_string_lossy() .to_string_lossy()
.to_string(); .to_string();
println!( println!("DEBUG: Storing collection path in Redis key 'collections:{}:path'", self.name);
"DEBUG: Storing collection path in Redis key 'collections:{}:path'", self.storage.store_collection_path(&self.name, &absolute_path)?;
self.name self.storage.store_collection_path(&self.name, &self.path.to_string_lossy())?;
);
self.storage
.store_collection_path(&self.name, &absolute_path)?;
self.storage
.store_collection_path(&self.name, &self.path.to_string_lossy())?;
// Walk through the directory // Walk through the directory
let walker = WalkDir::new(&self.path); let walker = WalkDir::new(&self.path);
@ -126,11 +116,11 @@ impl Collection {
// Determine if this is a document (markdown file) or an image // Determine if this is a document (markdown file) or an image
let is_markdown = filename.to_lowercase().ends_with(".md"); let is_markdown = filename.to_lowercase().ends_with(".md");
let is_image = filename.to_lowercase().ends_with(".png") let is_image = filename.to_lowercase().ends_with(".png") ||
|| filename.to_lowercase().ends_with(".jpg") filename.to_lowercase().ends_with(".jpg") ||
|| filename.to_lowercase().ends_with(".jpeg") filename.to_lowercase().ends_with(".jpeg") ||
|| filename.to_lowercase().ends_with(".gif") filename.to_lowercase().ends_with(".gif") ||
|| filename.to_lowercase().ends_with(".svg"); filename.to_lowercase().ends_with(".svg");
let file_type = if is_markdown { let file_type = if is_markdown {
"document" "document"
@ -142,19 +132,13 @@ impl Collection {
// Store in Redis using the namefixed filename as the key // Store in Redis using the namefixed filename as the key
// Store the original relative path to preserve case and special characters // Store the original relative path to preserve case and special characters
println!( println!("DEBUG: Storing {} '{}' in Redis key 'collections:{}' with key '{}' and value '{}'",
"DEBUG: Storing {} '{}' in Redis key 'collections:{}' with key '{}' and value '{}'", file_type, filename, self.name, namefixed_filename, rel_path.to_string_lossy());
file_type,
filename,
self.name,
namefixed_filename,
rel_path.to_string_lossy()
);
self.storage.store_collection_entry( self.storage.store_collection_entry(
&self.name, &self.name,
&namefixed_filename, &namefixed_filename,
&rel_path.to_string_lossy(), &rel_path.to_string_lossy()
)?; )?;
} }
@ -178,9 +162,7 @@ impl Collection {
let namefixed_page_name = ensure_md_extension(&namefixed_page_name); let namefixed_page_name = ensure_md_extension(&namefixed_page_name);
// Get the relative path from Redis // Get the relative path from Redis
let rel_path = self let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_page_name)
.storage
.get_collection_entry(&self.name, &namefixed_page_name)
.map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?; .map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?;
// Check if the path is valid // Check if the path is valid
@ -189,16 +171,14 @@ impl Collection {
// Return an error since the actual file path is not available // Return an error since the actual file path is not available
return Err(DocTreeError::IoError(std::io::Error::new( return Err(DocTreeError::IoError(std::io::Error::new(
std::io::ErrorKind::NotFound, std::io::ErrorKind::NotFound,
format!( format!("File path not available for {} in collection {}", page_name, self.name)
"File path not available for {} in collection {}",
page_name, self.name
),
))); )));
} }
// Read the file // Read the file
let full_path = self.path.join(rel_path); let full_path = self.path.join(rel_path);
let content = fs::read_to_string(full_path).map_err(|e| DocTreeError::IoError(e))?; let content = fs::read_to_string(full_path)
.map_err(|e| DocTreeError::IoError(e))?;
// Skip include processing at this level to avoid infinite recursion // Skip include processing at this level to avoid infinite recursion
// Include processing will be done at the higher level // Include processing will be done at the higher level
@ -235,11 +215,7 @@ impl Collection {
fs::write(&full_path, content).map_err(DocTreeError::IoError)?; fs::write(&full_path, content).map_err(DocTreeError::IoError)?;
// Update Redis // Update Redis
self.storage.store_collection_entry( self.storage.store_collection_entry(&self.name, &namefixed_page_name, &namefixed_page_name)?;
&self.name,
&namefixed_page_name,
&namefixed_page_name,
)?;
Ok(()) Ok(())
} }
@ -261,9 +237,7 @@ impl Collection {
let namefixed_page_name = ensure_md_extension(&namefixed_page_name); let namefixed_page_name = ensure_md_extension(&namefixed_page_name);
// Get the relative path from Redis // Get the relative path from Redis
let rel_path = self let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_page_name)
.storage
.get_collection_entry(&self.name, &namefixed_page_name)
.map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?; .map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?;
// Delete the file // Delete the file
@ -271,8 +245,7 @@ impl Collection {
fs::remove_file(full_path).map_err(DocTreeError::IoError)?; fs::remove_file(full_path).map_err(DocTreeError::IoError)?;
// Remove from Redis // Remove from Redis
self.storage self.storage.delete_collection_entry(&self.name, &namefixed_page_name)?;
.delete_collection_entry(&self.name, &namefixed_page_name)?;
Ok(()) Ok(())
} }
@ -287,8 +260,7 @@ impl Collection {
let keys = self.storage.list_collection_entries(&self.name)?; let keys = self.storage.list_collection_entries(&self.name)?;
// Filter to only include .md files // Filter to only include .md files
let pages = keys let pages = keys.into_iter()
.into_iter()
.filter(|key| key.ends_with(".md")) .filter(|key| key.ends_with(".md"))
.collect(); .collect();
@ -309,9 +281,7 @@ impl Collection {
let namefixed_file_name = name_fix(file_name); let namefixed_file_name = name_fix(file_name);
// Get the relative path from Redis // Get the relative path from Redis
let rel_path = self let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_file_name)
.storage
.get_collection_entry(&self.name, &namefixed_file_name)
.map_err(|_| DocTreeError::FileNotFound(file_name.to_string()))?; .map_err(|_| DocTreeError::FileNotFound(file_name.to_string()))?;
// Construct a URL for the file // Construct a URL for the file
@ -346,11 +316,7 @@ impl Collection {
fs::write(&full_path, content).map_err(DocTreeError::IoError)?; fs::write(&full_path, content).map_err(DocTreeError::IoError)?;
// Update Redis // Update Redis
self.storage.store_collection_entry( self.storage.store_collection_entry(&self.name, &namefixed_file_name, &namefixed_file_name)?;
&self.name,
&namefixed_file_name,
&namefixed_file_name,
)?;
Ok(()) Ok(())
} }
@ -369,9 +335,7 @@ impl Collection {
let namefixed_file_name = name_fix(file_name); let namefixed_file_name = name_fix(file_name);
// Get the relative path from Redis // Get the relative path from Redis
let rel_path = self let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_file_name)
.storage
.get_collection_entry(&self.name, &namefixed_file_name)
.map_err(|_| DocTreeError::FileNotFound(file_name.to_string()))?; .map_err(|_| DocTreeError::FileNotFound(file_name.to_string()))?;
// Delete the file // Delete the file
@ -379,8 +343,7 @@ impl Collection {
fs::remove_file(full_path).map_err(DocTreeError::IoError)?; fs::remove_file(full_path).map_err(DocTreeError::IoError)?;
// Remove from Redis // Remove from Redis
self.storage self.storage.delete_collection_entry(&self.name, &namefixed_file_name)?;
.delete_collection_entry(&self.name, &namefixed_file_name)?;
Ok(()) Ok(())
} }
@ -395,8 +358,7 @@ impl Collection {
let keys = self.storage.list_collection_entries(&self.name)?; let keys = self.storage.list_collection_entries(&self.name)?;
// Filter to exclude .md files // Filter to exclude .md files
let files = keys let files = keys.into_iter()
.into_iter()
.filter(|key| !key.ends_with(".md")) .filter(|key| !key.ends_with(".md"))
.collect(); .collect();
@ -420,8 +382,7 @@ impl Collection {
let namefixed_page_name = ensure_md_extension(&namefixed_page_name); let namefixed_page_name = ensure_md_extension(&namefixed_page_name);
// Get the relative path from Redis // Get the relative path from Redis
self.storage self.storage.get_collection_entry(&self.name, &namefixed_page_name)
.get_collection_entry(&self.name, &namefixed_page_name)
.map_err(|_| DocTreeError::PageNotFound(page_name.to_string())) .map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))
} }
@ -435,11 +396,7 @@ impl Collection {
/// # Returns /// # Returns
/// ///
/// The HTML content of the page or an error /// The HTML content of the page or an error
pub fn page_get_html( pub fn page_get_html(&self, page_name: &str, doctree: Option<&crate::doctree::DocTree>) -> Result<String> {
&self,
page_name: &str,
doctree: Option<&crate::doctree::DocTree>,
) -> Result<String> {
// Get the markdown content // Get the markdown content
let markdown = self.page_get(page_name)?; let markdown = self.page_get(page_name)?;
@ -479,8 +436,9 @@ impl Collection {
/// Ok(()) on success or an error. /// Ok(()) on success or an error.
pub fn export_to_ipfs(&self, output_csv_path: &Path) -> Result<()> { pub fn export_to_ipfs(&self, output_csv_path: &Path) -> Result<()> {
// Create a new tokio runtime and block on the async export function // Create a new tokio runtime and block on the async export function
tokio::runtime::Runtime::new()? tokio::runtime::Runtime::new()?.block_on(async {
.block_on(async { self.export_to_ipfs_async(output_csv_path).await })?; self.export_to_ipfs_async(output_csv_path).await
})?;
Ok(()) Ok(())
} }
@ -497,31 +455,25 @@ impl Collection {
pub async fn export_to_ipfs_async(&self, output_csv_path: &Path) -> Result<()> { pub async fn export_to_ipfs_async(&self, output_csv_path: &Path) -> Result<()> {
use blake3::Hasher; use blake3::Hasher;
// use chacha20poly1305::{ChaCha20Poly1305, Aead}; // use chacha20poly1305::{ChaCha20Poly1305, Aead};
use chacha20poly1305::aead::generic_array::GenericArray;
use csv::Writer;
use ipfs_api::IpfsClient; use ipfs_api::IpfsClient;
use rand::rngs::OsRng;
use tokio::fs::File; use tokio::fs::File;
use tokio::io::AsyncReadExt; use tokio::io::AsyncReadExt;
use csv::Writer;
use rand::rngs::OsRng;
use chacha20poly1305::aead::generic_array::GenericArray;
// Create the output directory if it doesn't exist // Create the output directory if it doesn't exist
// Create the output directory if it doesn't exist // Create the output directory if it doesn't exist
if let Some(parent) = output_csv_path.parent() { if let Some(parent) = output_csv_path.parent() {
if parent.exists() && parent.is_file() { if parent.exists() && parent.is_file() {
println!( println!("DEBUG: Removing conflicting file at output directory path: {:?}", parent);
"DEBUG: Removing conflicting file at output directory path: {:?}", tokio::fs::remove_file(parent).await.map_err(DocTreeError::IoError)?;
parent
);
tokio::fs::remove_file(parent)
.await
.map_err(DocTreeError::IoError)?;
println!("DEBUG: Conflicting file removed."); println!("DEBUG: Conflicting file removed.");
} }
if !parent.is_dir() { if !parent.is_dir() {
println!("DEBUG: Ensuring output directory exists: {:?}", parent); println!("DEBUG: Ensuring output directory exists: {:?}", parent);
tokio::fs::create_dir_all(parent) tokio::fs::create_dir_all(parent).await.map_err(DocTreeError::IoError)?;
.await
.map_err(DocTreeError::IoError)?;
println!("DEBUG: Output directory ensured."); println!("DEBUG: Output directory ensured.");
} else { } else {
println!("DEBUG: Output directory already exists: {:?}", parent); println!("DEBUG: Output directory already exists: {:?}", parent);
@ -529,10 +481,7 @@ impl Collection {
} }
// Create the CSV writer // Create the CSV writer
println!( println!("DEBUG: Creating or overwriting CSV file at {:?}", output_csv_path);
"DEBUG: Creating or overwriting CSV file at {:?}",
output_csv_path
);
let file = std::fs::OpenOptions::new() let file = std::fs::OpenOptions::new()
.write(true) .write(true)
.create(true) .create(true)
@ -543,15 +492,7 @@ impl Collection {
println!("DEBUG: CSV writer created successfully"); println!("DEBUG: CSV writer created successfully");
// Write the CSV header // Write the CSV header
writer writer.write_record(&["collectionname", "filename", "blakehash", "ipfshash", "size"]).map_err(|e| DocTreeError::CsvError(e.to_string()))?;
.write_record(&[
"collectionname",
"filename",
"blakehash",
"ipfshash",
"size",
])
.map_err(|e| DocTreeError::CsvError(e.to_string()))?;
// Connect to IPFS // Connect to IPFS
// let ipfs = IpfsClient::new("127.0.0.1:5001").await.map_err(|e| DocTreeError::IpfsError(e.to_string()))?; // let ipfs = IpfsClient::new("127.0.0.1:5001").await.map_err(|e| DocTreeError::IpfsError(e.to_string()))?;
@ -569,9 +510,7 @@ impl Collection {
for entry_name in entries { for entry_name in entries {
println!("DEBUG: Processing entry: {}", entry_name); println!("DEBUG: Processing entry: {}", entry_name);
// Get the relative path from Redis // Get the relative path from Redis
let relative_path = self let relative_path = self.storage.get_collection_entry(&self.name, &entry_name)
.storage
.get_collection_entry(&self.name, &entry_name)
.map_err(|_| DocTreeError::FileNotFound(entry_name.clone()))?; .map_err(|_| DocTreeError::FileNotFound(entry_name.clone()))?;
println!("DEBUG: Retrieved relative path: {}", relative_path); println!("DEBUG: Retrieved relative path: {}", relative_path);
@ -621,12 +560,9 @@ impl Collection {
println!("DEBUG: Adding file to IPFS: {:?}", file_path); println!("DEBUG: Adding file to IPFS: {:?}", file_path);
let ipfs_path = match ipfs.add(std::io::Cursor::new(content)).await { let ipfs_path = match ipfs.add(std::io::Cursor::new(content)).await {
Ok(path) => { Ok(path) => {
println!( println!("DEBUG: Successfully added file to IPFS. Hash: {}", path.hash);
"DEBUG: Successfully added file to IPFS. Hash: {}",
path.hash
);
path path
} },
Err(e) => { Err(e) => {
eprintln!("Error adding file to IPFS {:?}: {}", file_path, e); eprintln!("Error adding file to IPFS {:?}: {}", file_path, e);
continue; continue;
@ -652,9 +588,7 @@ impl Collection {
// Flush the CSV writer // Flush the CSV writer
println!("DEBUG: Flushing CSV writer"); println!("DEBUG: Flushing CSV writer");
writer writer.flush().map_err(|e| DocTreeError::CsvError(e.to_string()))?;
.flush()
.map_err(|e| DocTreeError::CsvError(e.to_string()))?;
println!("DEBUG: CSV writer flushed successfully"); println!("DEBUG: CSV writer flushed successfully");
Ok(()) Ok(())
@ -682,9 +616,9 @@ impl CollectionBuilder {
/// ///
/// A new Collection or an error /// A new Collection or an error
pub fn build(self) -> Result<Collection> { pub fn build(self) -> Result<Collection> {
let storage = self let storage = self.storage.ok_or_else(|| {
.storage DocTreeError::MissingParameter("storage".to_string())
.ok_or_else(|| DocTreeError::MissingParameter("storage".to_string()))?; })?;
let collection = Collection { let collection = Collection {
path: self.path, path: self.path,
@ -694,4 +628,4 @@ impl CollectionBuilder {
Ok(collection) Ok(collection)
} }
} }

173
impl_plan.md
View File

@ -1,173 +0,0 @@
# DocTree WebBuilder Implementation Plan
## Overview
This document outlines the implementation plan for the WebBuilder component of the DocTree project. The WebBuilder is designed to process hjson configuration files (like those in `examples/doctreenew/sites/demo1/`) and generate a `webmeta.json` file that can be used by a browser-based website generator.
## Current Status
### What's Implemented:
1. **DocTree Core Functionality**:
- The main DocTree library with functionality for scanning directories, managing collections, processing includes, and converting markdown to HTML
- Redis storage backend for storing document metadata
- Command-line interface (doctreecmd) for interacting with collections
2. **Example Structure for the New Approach**:
- Example hjson configuration files in `examples/doctreenew/sites/demo1/`
- This includes `main.hjson`, `header.hjson`, `footer.hjson`, `collection.hjson`, and `pages/mypages1.hjson`
3. **Specification Document**:
- Detailed specification in `webbuilder/src/builder/specs.md`
- Example output format in `webbuilder/src/builder/webmeta.json`
### What's Not Yet Implemented:
1. **WebBuilder Implementation**:
- The actual Rust code for the webbuilder component
2. **Hjson Parsing**:
- Code to parse the hjson files in the doctreenew directory
3. **Git Repository Integration**:
- Functionality to download referenced collections from Git repositories
4. **IPFS Export**:
- Complete functionality to export assets to IPFS
5. **Browser-Based Generator**:
- The browser-based website generator that would use the webmeta.json file
## Implementation Plan
### Phase 1: Core WebBuilder Implementation (2-3 weeks)
1. **Setup Project Structure**:
- Create necessary modules and files in `webbuilder/src/`
- Define main data structures and traits
2. **Implement Hjson Parsing**:
- Add hjson crate dependency
- Create parsers for each hjson file type (main, header, footer, collection, pages)
- Implement validation for hjson files
3. **Implement Site Structure Builder**:
- Create a module to combine parsed hjson data into a cohesive site structure
- Implement navigation generation based on page definitions
4. **Implement WebMeta Generator**:
- Create functionality to generate the webmeta.json file
- Ensure all required metadata is included
### Phase 2: Git Integration and Collection Processing (2 weeks)
1. **Implement Git Repository Integration**:
- Add git2 crate dependency
- Create functionality to clone/pull repositories based on collection.hjson
- Implement caching to avoid unnecessary downloads
2. **Integrate with DocTree Library**:
- Create an adapter to use DocTree functionality with hjson-defined collections
- Implement processing of includes between documents
3. **Implement Content Processing**:
- Create functionality to process markdown content
- Handle special directives or custom syntax
### Phase 3: IPFS Integration (2 weeks)
1. **Enhance IPFS Integration**:
- Complete the IPFS export functionality in DocTree
- Create a module to handle IPFS uploads
2. **Implement Asset Management**:
- Create functionality to track and process assets (images, CSS, etc.)
- Ensure proper IPFS linking
3. **Implement Content Hashing**:
- Add Blake hash calculation for content integrity verification
- Store hashes in webmeta.json
### Phase 4: CLI and Testing (1-2 weeks)
1. **Implement Command-Line Interface**:
- Create a CLI for the webbuilder
- Add commands for building, validating, and deploying sites
2. **Write Comprehensive Tests**:
- Unit tests for each component
- Integration tests for the full workflow
- Test with example sites
3. **Documentation**:
- Update README with usage instructions
- Create detailed API documentation
- Add examples and tutorials
### Phase 5: Browser-Based Generator (Optional, 3-4 weeks)
1. **Design Browser Component**:
- Create a JavaScript/TypeScript library to consume webmeta.json
- Design component architecture
2. **Implement Content Rendering**:
- Create components to render markdown content
- Implement navigation and site structure
3. **Implement IPFS Integration**:
- Add functionality to fetch content from IPFS
- Implement content verification using Blake hashes
4. **Create Demo Site**:
- Build a demo site using the browser-based generator
- Showcase features and capabilities
## Technical Details
### Key Dependencies
- **hjson**: For parsing hjson configuration files
- **git2**: For Git repository integration
- **ipfs-api**: For IPFS integration
- **blake3**: For content hashing
- **clap**: For command-line interface
- **tokio**: For async operations
### Data Flow
1. Parse hjson files from input directory
2. Download referenced Git repositories
3. Process content with DocTree
4. Export assets to IPFS
5. Generate webmeta.json
6. (Optional) Upload webmeta.json to IPFS
### Key Challenges
1. **Git Integration**: Handling authentication, rate limits, and large repositories
2. **IPFS Performance**: Optimizing IPFS uploads for large sites
3. **Content Processing**: Ensuring proper handling of includes and special syntax
4. **Browser Compatibility**: Ensuring the browser-based generator works across different browsers
## Milestones and Timeline
1. **Core WebBuilder Implementation**: Weeks 1-3
2. **Git Integration and Collection Processing**: Weeks 4-5
3. **IPFS Integration**: Weeks 6-7
4. **CLI and Testing**: Weeks 8-9
5. **Browser-Based Generator (Optional)**: Weeks 10-13
## Resources Required
1. **Development Resources**:
- 1-2 Rust developers
- 1 Frontend developer (for browser-based generator)
2. **Infrastructure**:
- IPFS node for testing
- Git repositories for testing
- CI/CD pipeline
## Conclusion
This implementation plan provides a roadmap for developing the WebBuilder component of the DocTree project. By following this plan, we can transform the current specification and example files into a fully functional system for generating websites from hjson configuration files and markdown content.

42
webbuilder/Cargo.toml
View File

@ -1,58 +1,24 @@
[package] [package]
name = "webbuilder" name = "doctree"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2024"
description = "A tool for building websites from hjson configuration files and markdown content"
authors = ["DocTree Team"]
[lib] [lib]
path = "src/lib.rs" path = "src/lib.rs"
[[bin]]
name = "webbuilder"
path = "src/main.rs"
[dependencies] [dependencies]
# Core dependencies
doctree = { path = "../doctree" }
walkdir = "2.3.3" walkdir = "2.3.3"
pulldown-cmark = "0.9.3" pulldown-cmark = "0.9.3"
thiserror = "1.0.40" thiserror = "1.0.40"
lazy_static = "1.4.0" lazy_static = "1.4.0"
toml = "0.7.3" toml = "0.7.3"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
redis = { version = "0.23.0", features = ["tokio-comp"] } redis = { version = "0.23.0", features = ["tokio-comp"] }
tokio = { version = "1.28.0", features = ["full"] } tokio = { version = "1.28.0", features = ["full"] }
sal = { git = "https://git.ourworld.tf/herocode/sal.git" } sal = { git = "https://git.ourworld.tf/herocode/sal.git" }
# Hjson parsing
deser-hjson = "1.1.0"
# Git integration is provided by the SAL library
# IPFS integration
ipfs-api-backend-hyper = "0.6"
ipfs-api = { version = "0.17.0", default-features = false, features = ["with-hyper-tls"] }
# Hashing and encryption
chacha20poly1305 = "0.10.1" chacha20poly1305 = "0.10.1"
blake3 = "1.3.1" blake3 = "1.3.1"
# CLI
clap = { version = "4.3.0", features = ["derive"] }
# Utilities
anyhow = "1.0.71"
log = "0.4.17"
env_logger = "0.10.0"
csv = "1.1" csv = "1.1"
rand = "0.9.1" rand = "0.9.1"
url = "2.3.1" ipfs-api-backend-hyper = "0.6"
ipfs-api = { version = "0.17.0", default-features = false, features = ["with-hyper-tls"] }
[dev-dependencies]
# Testing
tempfile = "3.5.0"
mockall = "0.11.4"
assert_fs = "1.0.10"
predicates = "3.0.3"

128
webbuilder/README.md
View File

@ -1,128 +0,0 @@
# WebBuilder
WebBuilder is a library for building websites from configuration files and markdown content. It uses the DocTree library to process markdown content and includes, and exports the result to a webmeta.json file that can be used by a browser-based website generator.
## Overview
WebBuilder scans directories for configuration files (in hjson format) and generates a `webmeta.json` file that can be used by a browser-based website generator. It can also clone Git repositories, process markdown content, and upload files to IPFS.
## Parsing Configuration Files
WebBuilder supports multiple parsing strategies for configuration files:
### Unified Parser
The recommended way to parse configuration files is to use the unified parser, which provides a consistent interface for all parsing strategies:
```rust
use webbuilder::{from_directory_with_strategy, ParsingStrategy};
// Use the recommended strategy (Hjson)
let webbuilder = from_directory_with_strategy("path/to/config", ParsingStrategy::Hjson)?;
// Or use the auto-detect strategy
let webbuilder = from_directory_with_strategy("path/to/config", ParsingStrategy::Auto)?;
// Or use the simple strategy (legacy)
let webbuilder = from_directory_with_strategy("path/to/config", ParsingStrategy::Simple)?;
```
You can also use the convenience functions:
```rust
use webbuilder::{from_directory, parse_site_config_recommended, parse_site_config_auto};
// Use the recommended strategy (Hjson)
let webbuilder = from_directory("path/to/config")?;
// Or parse the site configuration directly
let site_config = parse_site_config_recommended("path/to/config")?;
let site_config = parse_site_config_auto("path/to/config")?;
```
### Parsing Strategies
WebBuilder supports the following parsing strategies:
- **Hjson**: Uses the `deser-hjson` library to parse hjson files. This is the recommended strategy.
- **Simple**: Uses a simple line-by-line parser that doesn't rely on external libraries. This is a legacy strategy.
- **Auto**: Tries the Hjson parser first, and falls back to the simple parser if it fails.
## Building a Website
Once you have a WebBuilder instance, you can build a website:
```rust
use webbuilder::from_directory;
// Create a WebBuilder instance
let webbuilder = from_directory("path/to/config")?;
// Build the website
let webmeta = webbuilder.build()?;
// Save the webmeta.json file
webmeta.save("webmeta.json")?;
// Upload the webmeta.json file to IPFS
let ipfs_hash = webbuilder.upload_to_ipfs("webmeta.json")?;
println!("Uploaded to IPFS: {}", ipfs_hash);
```
## Configuration Files
WebBuilder expects the following configuration files:
- `main.hjson`: Main configuration file with site metadata
- `header.hjson`: Header configuration
- `footer.hjson`: Footer configuration
- `collection.hjson`: Collection configuration (Git repositories)
- `pages/*.hjson`: Page configuration files
Example `main.hjson`:
```hjson
{
"name": "my-site",
"title": "My Site",
"description": "My awesome site",
"url": "https://example.com",
"favicon": "favicon.ico",
"keywords": [
"website",
"awesome"
]
}
```
Example `collection.hjson`:
```hjson
[
{
"name": "docs",
"url": "https://github.com/example/docs.git",
"description": "Documentation",
"scan": true
}
]
```
Example `pages/pages.hjson`:
```hjson
[
{
"name": "home",
"title": "Home",
"description": "Home page",
"navpath": "/",
"collection": "docs",
"draft": false
}
]
```
## License
This project is licensed under the MIT License - see the LICENSE file for details.

324
webbuilder/src/builder/mod.rs
View File

@ -1,324 +0,0 @@
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::Path;
use crate::config::SiteConfig;
use crate::error::Result;
use crate::parser;
#[cfg(test)]
mod mod_test;
/// WebMeta represents the output of the WebBuilder
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebMeta {
/// Site metadata
pub site_metadata: SiteMetadata,
/// Pages
pub pages: Vec<PageMeta>,
/// Assets
pub assets: std::collections::HashMap<String, AssetMeta>,
}
/// Site metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SiteMetadata {
/// Site name
pub name: String,
/// Site title
pub title: String,
/// Site description
pub description: Option<String>,
/// Site keywords
pub keywords: Option<Vec<String>>,
/// Site header
pub header: Option<serde_json::Value>,
/// Site footer
pub footer: Option<serde_json::Value>,
}
/// Page metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PageMeta {
/// Page ID
pub id: String,
/// Page title
pub title: String,
/// IPFS key of the page content
pub ipfs_key: String,
/// Blake hash of the page content
pub blakehash: String,
/// Page sections
pub sections: Vec<SectionMeta>,
/// Page assets
pub assets: Vec<AssetMeta>,
}
/// Section metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SectionMeta {
/// Section type
#[serde(rename = "type")]
pub section_type: String,
/// Section content
pub content: String,
}
/// Asset metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AssetMeta {
/// Asset name
pub name: String,
/// IPFS key of the asset
pub ipfs_key: String,
}
impl WebMeta {
/// Save the WebMeta to a file
///
/// # Arguments
///
/// * `path` - Path to save the file to
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {
let json = serde_json::to_string_pretty(self)?;
fs::write(path, json)?;
Ok(())
}
}
/// WebBuilder is responsible for building a website from hjson configuration files
#[derive(Debug)]
pub struct WebBuilder {
/// Site configuration
pub config: SiteConfig,
}
impl WebBuilder {
/// Create a new WebBuilder instance from a directory containing hjson configuration files
///
/// # Arguments
///
/// * `path` - Path to the directory containing hjson configuration files
///
/// # Returns
///
/// A new WebBuilder instance or an error
pub fn from_directory<P: AsRef<Path>>(path: P) -> Result<Self> {
let config = parser::parse_site_config_recommended(path)?;
Ok(WebBuilder { config })
}
/// Build the website
///
/// # Returns
///
/// A WebMeta instance or an error
pub fn build(&self) -> Result<WebMeta> {
// Create site metadata
let site_metadata = SiteMetadata {
name: self.config.name.clone(),
title: self.config.title.clone(),
description: self.config.description.clone(),
keywords: self.config.keywords.clone(),
header: self
.config
.header
.as_ref()
.map(|h| serde_json::to_value(h).unwrap_or_default()),
footer: self
.config
.footer
.as_ref()
.map(|f| serde_json::to_value(f).unwrap_or_default()),
};
// Process collections
let mut pages = Vec::new();
let assets = std::collections::HashMap::new();
// Process collections from Git repositories
for collection in &self.config.collections {
if let Some(url) = &collection.url {
// Extract repository name from URL
let repo_name = collection.name.clone().unwrap_or_else(|| {
url.split('/')
.last()
.unwrap_or("repo")
.trim_end_matches(".git")
.to_string()
});
// Clone or pull the Git repository
let repo_path = self.config.base_path.join("repos").join(&repo_name);
// Create the repos directory if it doesn't exist
if !repo_path.parent().unwrap().exists() {
fs::create_dir_all(repo_path.parent().unwrap())?;
}
// Clone or pull the repository
let repo_path = match crate::git::clone_or_pull(url, &repo_path) {
Ok(path) => path,
Err(e) => {
// Log the error but continue with a placeholder
log::warn!("Failed to clone repository {}: {}", url, e);
// Create a placeholder page for the failed repository
let page_id = format!("{}-index", repo_name);
let page = PageMeta {
id: page_id.clone(),
title: format!("{} Index", repo_name),
ipfs_key: "QmPlaceholderIpfsKey".to_string(),
blakehash: "blake3-placeholder".to_string(),
sections: vec![SectionMeta {
section_type: "markdown".to_string(),
content: format!(
"# {} Index\n\nFailed to clone repository: {}\nURL: {}",
repo_name, e, url
),
}],
assets: Vec::new(),
};
pages.push(page);
continue;
}
};
// Create a page for the repository
let page_id = format!("{}-index", repo_name);
let page = PageMeta {
id: page_id.clone(),
title: format!("{} Index", repo_name),
ipfs_key: "QmPlaceholderIpfsKey".to_string(), // Will be replaced with actual IPFS key
blakehash: "blake3-placeholder".to_string(), // Will be replaced with actual Blake hash
sections: vec![SectionMeta {
section_type: "markdown".to_string(),
content: format!(
"# {} Index\n\nRepository cloned successfully.\nPath: {}\nURL: {}",
repo_name, repo_path.display(), url
),
}],
assets: Vec::new(),
};
pages.push(page);
}
}
// Process pages from the configuration
for page_config in &self.config.pages {
// Skip draft pages unless explicitly set to false
if page_config.draft.unwrap_or(false) {
log::info!("Skipping draft page: {}", page_config.name);
continue;
}
// Generate a unique page ID
let page_id = format!("page-{}", page_config.name);
// Find the collection for this page
let collection_path = self.config.collections.iter()
.find(|c| c.name.as_ref().map_or(false, |name| name == &page_config.collection))
.and_then(|c| c.url.as_ref())
.map(|url| {
let repo_name = url.split('/')
.last()
.unwrap_or("repo")
.trim_end_matches(".git")
.to_string();
self.config.base_path.join("repos").join(&repo_name)
});
// Create the page content
let content = if let Some(collection_path) = collection_path {
// Try to find the page content in the collection
let page_path = collection_path.join(&page_config.name).with_extension("md");
if page_path.exists() {
match fs::read_to_string(&page_path) {
Ok(content) => content,
Err(e) => {
log::warn!("Failed to read page content from {}: {}", page_path.display(), e);
format!(
"# {}\n\n{}\n\n*Failed to read page content from {}*",
page_config.title,
page_config.description.clone().unwrap_or_default(),
page_path.display()
)
}
}
} else {
format!(
"# {}\n\n{}\n\n*Page content not found at {}*",
page_config.title,
page_config.description.clone().unwrap_or_default(),
page_path.display()
)
}
} else {
format!(
"# {}\n\n{}",
page_config.title,
page_config.description.clone().unwrap_or_default()
)
};
// Calculate the Blake hash of the content
let content_bytes = content.as_bytes();
let blakehash = format!("blake3-{}", blake3::hash(content_bytes).to_hex());
// Create the page metadata
let page = PageMeta {
id: page_id.clone(),
title: page_config.title.clone(),
ipfs_key: "QmPlaceholderIpfsKey".to_string(), // Will be replaced with actual IPFS key
blakehash,
sections: vec![SectionMeta {
section_type: "markdown".to_string(),
content,
}],
assets: Vec::new(),
};
pages.push(page);
}
// Create the WebMeta
Ok(WebMeta {
site_metadata,
pages,
assets,
})
}
/// Upload a file to IPFS
///
/// # Arguments
///
/// * `path` - Path to the file to upload
///
/// # Returns
///
/// The IPFS hash of the file or an error
pub fn upload_to_ipfs<P: AsRef<Path>>(&self, path: P) -> Result<String> {
crate::ipfs::upload_file(path)
}
}

200
webbuilder/src/builder/mod_test.rs
View File

@ -1,200 +0,0 @@
#[cfg(test)]
mod tests {
use crate::builder::{PageMeta, SectionMeta, SiteMetadata, WebMeta};
use crate::config::{CollectionConfig, PageConfig, SiteConfig};
use crate::error::WebBuilderError;
use crate::WebBuilder;
use std::fs;
use std::path::PathBuf;
use tempfile::TempDir;
fn create_test_config() -> SiteConfig {
SiteConfig {
name: "test".to_string(),
title: "Test Site".to_string(),
description: Some("A test site".to_string()),
keywords: Some(vec!["test".to_string(), "site".to_string()]),
url: Some("https://example.com".to_string()),
favicon: Some("favicon.ico".to_string()),
header: None,
footer: None,
collections: vec![CollectionConfig {
name: Some("test".to_string()),
url: Some("https://git.ourworld.tf/tfgrid/home.git".to_string()),
description: Some("A test collection".to_string()),
scan: Some(true),
}],
pages: vec![PageConfig {
name: "home".to_string(),
title: "Home".to_string(),
description: Some("Home page".to_string()),
navpath: "/".to_string(),
collection: "test".to_string(),
draft: Some(false),
}],
base_path: PathBuf::from("/path/to/site"),
}
}
#[test]
fn test_webmeta_save() {
let temp_dir = TempDir::new().unwrap();
let output_path = temp_dir.path().join("webmeta.json");
let webmeta = WebMeta {
site_metadata: SiteMetadata {
name: "test".to_string(),
title: "Test Site".to_string(),
description: Some("A test site".to_string()),
keywords: Some(vec!["test".to_string(), "site".to_string()]),
header: None,
footer: None,
},
pages: vec![PageMeta {
id: "page-1".to_string(),
title: "Page 1".to_string(),
ipfs_key: "QmTest1".to_string(),
blakehash: "blake3-test1".to_string(),
sections: vec![SectionMeta {
section_type: "markdown".to_string(),
content: "# Page 1\n\nThis is page 1.".to_string(),
}],
assets: vec![],
}],
assets: std::collections::HashMap::new(),
};
// Save the webmeta.json file
webmeta.save(&output_path).unwrap();
// Check that the file exists
assert!(output_path.exists());
// Read the file and parse it
let content = fs::read_to_string(&output_path).unwrap();
let parsed: WebMeta = serde_json::from_str(&content).unwrap();
// Check that the parsed webmeta matches the original
assert_eq!(parsed.site_metadata.name, webmeta.site_metadata.name);
assert_eq!(parsed.site_metadata.title, webmeta.site_metadata.title);
assert_eq!(
parsed.site_metadata.description,
webmeta.site_metadata.description
);
assert_eq!(
parsed.site_metadata.keywords,
webmeta.site_metadata.keywords
);
assert_eq!(parsed.pages.len(), webmeta.pages.len());
assert_eq!(parsed.pages[0].id, webmeta.pages[0].id);
assert_eq!(parsed.pages[0].title, webmeta.pages[0].title);
assert_eq!(parsed.pages[0].ipfs_key, webmeta.pages[0].ipfs_key);
assert_eq!(parsed.pages[0].blakehash, webmeta.pages[0].blakehash);
assert_eq!(
parsed.pages[0].sections.len(),
webmeta.pages[0].sections.len()
);
assert_eq!(
parsed.pages[0].sections[0].section_type,
webmeta.pages[0].sections[0].section_type
);
assert_eq!(
parsed.pages[0].sections[0].content,
webmeta.pages[0].sections[0].content
);
}
#[test]
fn test_webbuilder_build() {
// Create a temporary directory for the test
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().to_path_buf();
// Create a modified test config with the temporary directory as base_path
let mut config = create_test_config();
config.base_path = site_dir.clone();
// Create the repos directory
let repos_dir = site_dir.join("repos");
fs::create_dir_all(&repos_dir).unwrap();
// Create a mock repository directory
let repo_dir = repos_dir.join("home");
fs::create_dir_all(&repo_dir).unwrap();
// Create a mock page file in the repository
let page_content = "# Home Page\n\nThis is the home page content.";
fs::write(repo_dir.join("home.md"), page_content).unwrap();
// Create the WebBuilder with our config
let webbuilder = WebBuilder { config };
// Mock the git module to avoid actual git operations
// This is a simplified test that assumes the git operations would succeed
// Build the website
let webmeta = webbuilder.build().unwrap();
// Check site metadata
assert_eq!(webmeta.site_metadata.name, "test");
assert_eq!(webmeta.site_metadata.title, "Test Site");
assert_eq!(
webmeta.site_metadata.description,
Some("A test site".to_string())
);
assert_eq!(
webmeta.site_metadata.keywords,
Some(vec!["test".to_string(), "site".to_string()])
);
// We expect at least one page from the configuration
assert!(webmeta.pages.len() >= 1);
// Find the page with ID "page-home"
let home_page = webmeta.pages.iter().find(|p| p.id == "page-home");
// Check that we found the page
assert!(home_page.is_some());
let home_page = home_page.unwrap();
// Check the page properties
assert_eq!(home_page.title, "Home");
assert_eq!(home_page.ipfs_key, "QmPlaceholderIpfsKey");
assert_eq!(home_page.sections.len(), 1);
assert_eq!(home_page.sections[0].section_type, "markdown");
// The content should either be our mock content or a placeholder
// depending on whether the page was found
assert!(
home_page.sections[0].content.contains("Home") ||
home_page.sections[0].content.contains("home.md")
);
}
#[test]
fn test_webbuilder_from_directory() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create main.hjson
let main_hjson = r#"{ "name": "test", "title": "Test Site" }"#;
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
let webbuilder = WebBuilder::from_directory(&site_dir).unwrap();
assert_eq!(webbuilder.config.name, "test");
assert_eq!(webbuilder.config.title, "Test Site");
}
#[test]
fn test_webbuilder_from_directory_error() {
let result = WebBuilder::from_directory("/nonexistent/directory");
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
WebBuilderError::MissingDirectory(_)
));
}
}

43
webbuilder/src/builder/webmeta.json Normal file
View File

@ -0,0 +1,43 @@
{
"site_metadata": {
"name": "demo1",
"title": "Demo Site 1",
"description": "This is a demo site for doctree",
"keywords": ["demo", "doctree", "example"],
"header": {
"logo": "/images/logo.png",
"nav": [
{ "text": "Home", "url": "/" },
{ "text": "About", "url": "/about" }
]
},
"footer": {
"copyright": "© 2023 My Company",
"links": [
{ "text": "Privacy Policy", "url": "/privacy" }
]
}
},
"pages": [
{
"id": "mypages1",
"title": "My Pages 1",
"ipfs_key": "QmPlaceholderIpfsKey1",
"blakehash": "sha256-PlaceholderBlakeHash1",
"sections": [
{ "type": "text", "content": "This is example content for My Pages 1." }
],
"assets": [
{
"name": "image1.png",
"ipfs_key": "QmPlaceholderImageIpfsKey1"
}
]
}
],
"assets": {
"style.css": {
"ipfs_key": "QmPlaceholderCssIpfsKey1"
}
}
}

214
webbuilder/src/config.rs
View File

@ -1,214 +0,0 @@
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use crate::error::{Result, WebBuilderError};
/// Site configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SiteConfig {
/// Site name
pub name: String,
/// Site title
pub title: String,
/// Site description
pub description: Option<String>,
/// Site keywords
pub keywords: Option<Vec<String>>,
/// Site URL
pub url: Option<String>,
/// Site favicon
pub favicon: Option<String>,
/// Site header
pub header: Option<HeaderConfig>,
/// Site footer
pub footer: Option<FooterConfig>,
/// Site collections
pub collections: Vec<CollectionConfig>,
/// Site pages
pub pages: Vec<PageConfig>,
/// Base path of the site configuration
#[serde(skip)]
pub base_path: PathBuf,
}
/// Header configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HeaderConfig {
/// Header logo
pub logo: Option<LogoConfig>,
/// Header title
pub title: Option<String>,
/// Header menu
pub menu: Option<Vec<MenuItemConfig>>,
/// Login button
pub login: Option<LoginConfig>,
}
/// Logo configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LogoConfig {
/// Logo source
pub src: String,
/// Logo alt text
pub alt: Option<String>,
}
/// Menu item configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MenuItemConfig {
/// Menu item label
pub label: String,
/// Menu item link
pub link: String,
/// Menu item children
pub children: Option<Vec<MenuItemConfig>>,
}
/// Login button configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LoginConfig {
/// Whether the login button is visible
pub visible: bool,
/// Login button label
pub label: Option<String>,
/// Login button link
pub link: Option<String>,
}
/// Footer configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FooterConfig {
/// Footer title
pub title: Option<String>,
/// Footer sections
pub sections: Option<Vec<FooterSectionConfig>>,
/// Footer copyright
pub copyright: Option<String>,
}
/// Footer section configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FooterSectionConfig {
/// Section title
pub title: String,
/// Section links
pub links: Vec<LinkConfig>,
}
/// Link configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LinkConfig {
/// Link label
pub label: String,
/// Link URL
pub href: String,
}
/// Collection configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CollectionConfig {
/// Collection name
pub name: Option<String>,
/// Collection URL
pub url: Option<String>,
/// Collection description
pub description: Option<String>,
/// Whether to scan the URL for collections
pub scan: Option<bool>,
}
/// Page configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PageConfig {
/// Page name
pub name: String,
/// Page title
pub title: String,
/// Page description
pub description: Option<String>,
/// Page navigation path
pub navpath: String,
/// Page collection
pub collection: String,
/// Whether the page is a draft
pub draft: Option<bool>,
}
impl SiteConfig {
/// Load site configuration from a directory
///
/// # Arguments
///
/// * `path` - Path to the directory containing hjson configuration files
///
/// # Returns
///
/// A new SiteConfig instance or an error
pub fn from_directory<P: AsRef<Path>>(path: P) -> Result<Self> {
let path = path.as_ref();
// Check if the directory exists
if !path.exists() {
return Err(WebBuilderError::MissingDirectory(path.to_path_buf()));
}
// Check if the directory is a directory
if !path.is_dir() {
return Err(WebBuilderError::InvalidConfiguration(format!(
"{:?} is not a directory",
path
)));
}
// TODO: Implement loading configuration from hjson files
// For now, return a placeholder configuration
Ok(SiteConfig {
name: "demo1".to_string(),
title: "Demo Site 1".to_string(),
description: Some("This is a demo site for doctree".to_string()),
keywords: Some(vec![
"demo".to_string(),
"doctree".to_string(),
"example".to_string(),
]),
url: Some("https://example.com".to_string()),
favicon: Some("img/favicon.png".to_string()),
header: None,
footer: None,
collections: Vec::new(),
pages: Vec::new(),
base_path: path.to_path_buf(),
})
}
}

156
webbuilder/src/config_test.rs
View File

@ -1,156 +0,0 @@
#[cfg(test)]
mod tests {
use crate::config::{
CollectionConfig, FooterConfig, FooterSectionConfig, HeaderConfig, LinkConfig, LoginConfig,
LogoConfig, MenuItemConfig, PageConfig, SiteConfig,
};
use std::path::PathBuf;
#[test]
fn test_site_config_serialization() {
let config = SiteConfig {
name: "test".to_string(),
title: "Test Site".to_string(),
description: Some("A test site".to_string()),
keywords: Some(vec!["test".to_string(), "site".to_string()]),
url: Some("https://example.com".to_string()),
favicon: Some("favicon.ico".to_string()),
header: Some(HeaderConfig {
logo: Some(LogoConfig {
src: "logo.png".to_string(),
alt: Some("Logo".to_string()),
}),
title: Some("Test Site".to_string()),
menu: Some(vec![
MenuItemConfig {
label: "Home".to_string(),
link: "/".to_string(),
children: None,
},
MenuItemConfig {
label: "About".to_string(),
link: "/about".to_string(),
children: Some(vec![MenuItemConfig {
label: "Team".to_string(),
link: "/about/team".to_string(),
children: None,
}]),
},
]),
login: Some(LoginConfig {
visible: true,
label: Some("Login".to_string()),
link: Some("/login".to_string()),
}),
}),
footer: Some(FooterConfig {
title: Some("Test Site".to_string()),
sections: Some(vec![FooterSectionConfig {
title: "Links".to_string(),
links: vec![
LinkConfig {
label: "Home".to_string(),
href: "/".to_string(),
},
LinkConfig {
label: "About".to_string(),
href: "/about".to_string(),
},
],
}]),
copyright: Some("© 2023".to_string()),
}),
collections: vec![CollectionConfig {
name: Some("test".to_string()),
url: Some("https://git.ourworld.tf/tfgrid/home.git".to_string()),
description: Some("A test collection".to_string()),
scan: Some(true),
}],
pages: vec![PageConfig {
name: "home".to_string(),
title: "Home".to_string(),
description: Some("Home page".to_string()),
navpath: "/".to_string(),
collection: "test".to_string(),
draft: Some(false),
}],
base_path: PathBuf::from("/path/to/site"),
};
// Serialize to JSON
let json = serde_json::to_string(&config).unwrap();
// Deserialize from JSON
let deserialized: SiteConfig = serde_json::from_str(&json).unwrap();
// Check that the deserialized config matches the original
assert_eq!(deserialized.name, config.name);
assert_eq!(deserialized.title, config.title);
assert_eq!(deserialized.description, config.description);
assert_eq!(deserialized.keywords, config.keywords);
assert_eq!(deserialized.url, config.url);
assert_eq!(deserialized.favicon, config.favicon);
// Check header
assert!(deserialized.header.is_some());
let header = deserialized.header.as_ref().unwrap();
let original_header = config.header.as_ref().unwrap();
// Check logo
assert!(header.logo.is_some());
let logo = header.logo.as_ref().unwrap();
let original_logo = original_header.logo.as_ref().unwrap();
assert_eq!(logo.src, original_logo.src);
assert_eq!(logo.alt, original_logo.alt);
// Check title
assert_eq!(header.title, original_header.title);
// Check menu
assert!(header.menu.is_some());
let menu = header.menu.as_ref().unwrap();
let original_menu = original_header.menu.as_ref().unwrap();
assert_eq!(menu.len(), original_menu.len());
assert_eq!(menu[0].label, original_menu[0].label);
assert_eq!(menu[0].link, original_menu[0].link);
assert_eq!(menu[1].label, original_menu[1].label);
assert_eq!(menu[1].link, original_menu[1].link);
// Check login
assert!(header.login.is_some());
let login = header.login.as_ref().unwrap();
let original_login = original_header.login.as_ref().unwrap();
assert_eq!(login.visible, original_login.visible);
assert_eq!(login.label, original_login.label);
assert_eq!(login.link, original_login.link);
// Check footer
assert!(deserialized.footer.is_some());
let footer = deserialized.footer.as_ref().unwrap();
let original_footer = config.footer.as_ref().unwrap();
assert_eq!(footer.title, original_footer.title);
assert_eq!(footer.copyright, original_footer.copyright);
// Check collections
assert_eq!(deserialized.collections.len(), config.collections.len());
assert_eq!(deserialized.collections[0].name, config.collections[0].name);
assert_eq!(deserialized.collections[0].url, config.collections[0].url);
assert_eq!(
deserialized.collections[0].description,
config.collections[0].description
);
assert_eq!(deserialized.collections[0].scan, config.collections[0].scan);
// Check pages
assert_eq!(deserialized.pages.len(), config.pages.len());
assert_eq!(deserialized.pages[0].name, config.pages[0].name);
assert_eq!(deserialized.pages[0].title, config.pages[0].title);
assert_eq!(
deserialized.pages[0].description,
config.pages[0].description
);
assert_eq!(deserialized.pages[0].navpath, config.pages[0].navpath);
assert_eq!(deserialized.pages[0].collection, config.pages[0].collection);
assert_eq!(deserialized.pages[0].draft, config.pages[0].draft);
}
}

68
webbuilder/src/error.rs
View File

@ -1,68 +0,0 @@
use std::io;
use std::path::PathBuf;
use thiserror::Error;
/// Result type for WebBuilder operations
pub type Result<T> = std::result::Result<T, WebBuilderError>;
/// Error type for WebBuilder operations
#[derive(Error, Debug)]
pub enum WebBuilderError {
/// IO error
#[error("IO error: {0}")]
IoError(#[from] io::Error),
/// DocTree error
#[error("DocTree error: {0}")]
DocTreeError(#[from] doctree::DocTreeError),
/// Hjson parsing error
#[error("Hjson parsing error: {0}")]
HjsonError(String),
/// Git error
#[error("Git error: {0}")]
GitError(String),
/// IPFS error
#[error("IPFS error: {0}")]
IpfsError(String),
/// Missing file error
#[error("Missing file: {0}")]
MissingFile(PathBuf),
/// Missing directory error
#[error("Missing directory: {0}")]
MissingDirectory(PathBuf),
/// Missing configuration error
#[error("Missing configuration: {0}")]
MissingConfiguration(String),
/// Invalid configuration error
#[error("Invalid configuration: {0}")]
InvalidConfiguration(String),
/// Other error
#[error("Error: {0}")]
Other(String),
}
impl From<String> for WebBuilderError {
fn from(error: String) -> Self {
WebBuilderError::Other(error)
}
}
impl From<&str> for WebBuilderError {
fn from(error: &str) -> Self {
WebBuilderError::Other(error.to_string())
}
}
impl From<serde_json::Error> for WebBuilderError {
fn from(error: serde_json::Error) -> Self {
WebBuilderError::Other(format!("JSON error: {}", error))
}
}

73
webbuilder/src/error_test.rs
View File

@ -1,73 +0,0 @@
#[cfg(test)]
mod tests {
use crate::error::WebBuilderError;
use std::path::PathBuf;
#[test]
fn test_error_from_string() {
let error = WebBuilderError::from("test error");
assert!(matches!(error, WebBuilderError::Other(s) if s == "test error"));
}
#[test]
fn test_error_from_string_owned() {
let error = WebBuilderError::from("test error".to_string());
assert!(matches!(error, WebBuilderError::Other(s) if s == "test error"));
}
#[test]
fn test_error_from_json_error() {
let json_error = serde_json::from_str::<serde_json::Value>("invalid json").unwrap_err();
let error = WebBuilderError::from(json_error);
assert!(matches!(error, WebBuilderError::Other(s) if s.starts_with("JSON error:")));
}
#[test]
fn test_error_display() {
let errors = vec![
(
WebBuilderError::IoError(std::io::Error::new(
std::io::ErrorKind::NotFound,
"file not found",
)),
"IO error: file not found",
),
(
WebBuilderError::HjsonError("invalid hjson".to_string()),
"Hjson parsing error: invalid hjson",
),
(
WebBuilderError::GitError("git error".to_string()),
"Git error: git error",
),
(
WebBuilderError::IpfsError("ipfs error".to_string()),
"IPFS error: ipfs error",
),
(
WebBuilderError::MissingFile(PathBuf::from("/path/to/file")),
"Missing file: /path/to/file",
),
(
WebBuilderError::MissingDirectory(PathBuf::from("/path/to/dir")),
"Missing directory: /path/to/dir",
),
(
WebBuilderError::MissingConfiguration("config".to_string()),
"Missing configuration: config",
),
(
WebBuilderError::InvalidConfiguration("invalid config".to_string()),
"Invalid configuration: invalid config",
),
(
WebBuilderError::Other("other error".to_string()),
"Error: other error",
),
];
for (error, expected) in errors {
assert_eq!(error.to_string(), expected);
}
}
}

182
webbuilder/src/git.rs
View File

@ -1,182 +0,0 @@
use lazy_static::lazy_static;
use sal::git::{GitRepo, GitTree};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{ SystemTime};
use crate::error::{Result, WebBuilderError};
// Cache entry for Git repositories
struct CacheEntry {
path: PathBuf,
last_updated: SystemTime,
}
// Global cache for Git repositories
lazy_static! {
static ref REPO_CACHE: Arc<Mutex<HashMap<String, CacheEntry>>> =
Arc::new(Mutex::new(HashMap::new()));
}
// Cache timeout in seconds (default: 1 hour)
const CACHE_TIMEOUT: u64 = 3600;
/// Clone a Git repository
///
/// # Arguments
///
/// * `url` - URL of the repository to clone
/// * `destination` - Destination directory
///
/// # Returns
///
/// The path to the cloned repository or an error
pub fn clone_repository<P: AsRef<Path>>(url: &str, destination: P) -> Result<PathBuf> {
let destination = destination.as_ref();
let destination_str = destination.to_str().unwrap();
// Create a GitTree for the parent directory
let parent_dir = destination.parent().ok_or_else(|| {
WebBuilderError::InvalidConfiguration(format!(
"Invalid destination path: {}",
destination_str
))
})?;
let git_tree = GitTree::new(parent_dir.to_str().unwrap())
.map_err(|e| WebBuilderError::GitError(format!("Failed to create GitTree: {}", e)))?;
// Use the GitTree to get (clone) the repository
let repos = git_tree
.get(url)
.map_err(|e| WebBuilderError::GitError(format!("Failed to clone repository: {}", e)))?;
if repos.is_empty() {
return Err(WebBuilderError::GitError(format!(
"Failed to clone repository: No repository was created"
)));
}
// Return the path of the first repository
Ok(PathBuf::from(repos[0].path()))
}
/// Pull the latest changes from a Git repository
///
/// # Arguments
///
/// * `path` - Path to the repository
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn pull_repository<P: AsRef<Path>>(path: P) -> Result<()> {
let path = path.as_ref();
let path_str = path.to_str().unwrap();
// Create a GitRepo directly
let repo = GitRepo::new(path_str.to_string());
// Pull the repository
repo.pull()
.map_err(|e| WebBuilderError::GitError(format!("Failed to pull repository: {}", e)))?;
Ok(())
}
/// Clone or pull a Git repository with caching
///
/// # Arguments
///
/// * `url` - URL of the repository to clone
/// * `destination` - Destination directory
///
/// # Returns
///
/// The path to the repository or an error
pub fn clone_or_pull<P: AsRef<Path>>(url: &str, destination: P) -> Result<PathBuf> {
let destination = destination.as_ref();
// Check the cache first
let mut cache = REPO_CACHE.lock().unwrap();
let now = SystemTime::now();
if let Some(entry) = cache.get(url) {
// Check if the cache entry is still valid
if let Ok(elapsed) = now.duration_since(entry.last_updated) {
if elapsed.as_secs() < CACHE_TIMEOUT {
// Cache is still valid, return the cached path
log::info!("Using cached repository for {}", url);
return Ok(entry.path.clone());
}
}
}
// Cache miss or expired, clone or pull the repository
let result = if destination.exists() {
// Pull the repository
pull_repository(destination)?;
Ok(destination.to_path_buf())
} else {
// Clone the repository
clone_repository(url, destination)
};
// Update the cache
if let Ok(path) = &result {
cache.insert(
url.to_string(),
CacheEntry {
path: path.clone(),
last_updated: now,
},
);
}
result
}
/// Force update a Git repository, bypassing the cache
///
/// # Arguments
///
/// * `url` - URL of the repository to clone
/// * `destination` - Destination directory
///
/// # Returns
///
/// The path to the repository or an error
pub fn force_update<P: AsRef<Path>>(url: &str, destination: P) -> Result<PathBuf> {
let destination = destination.as_ref();
// Clone or pull the repository
let result = if destination.exists() {
// Pull the repository
pull_repository(destination)?;
Ok(destination.to_path_buf())
} else {
// Clone the repository
clone_repository(url, destination)
};
// Update the cache
if let Ok(path) = &result {
let mut cache = REPO_CACHE.lock().unwrap();
cache.insert(
url.to_string(),
CacheEntry {
path: path.clone(),
last_updated: SystemTime::now(),
},
);
}
result
}
/// Clear the Git repository cache
pub fn clear_cache() {
let mut cache = REPO_CACHE.lock().unwrap();
cache.clear();
}

26
webbuilder/src/git_test.rs
View File

@ -1,26 +0,0 @@
#[cfg(test)]
mod tests {
use crate::error::WebBuilderError;
use crate::git::clone_repository;
use std::path::PathBuf;
#[test]
fn test_clone_repository_error_invalid_destination() {
// Test with a destination that has no parent directory
// This URL is invalid because we added number 2 after `home`
let result = clone_repository("https://git.ourworld.tf/tfgrid/home2.git", PathBuf::from("/"));
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
WebBuilderError::InvalidConfiguration(_)
));
}
// Note: The following tests would require mocking the sal::git module,
// which is complex due to the external dependency. In a real-world scenario,
// we would use a more sophisticated mocking approach or integration tests.
// For now, we'll just test the error cases and leave the success cases
// for integration testing.
}

70
webbuilder/src/ipfs.rs
View File

@ -1,70 +0,0 @@
use ipfs_api_backend_hyper::{IpfsApi, IpfsClient};
use std::fs::File;
use std::path::Path;
use tokio::runtime::Runtime;
use crate::error::{Result, WebBuilderError};
/// Upload a file to IPFS
///
/// # Arguments
///
/// * `path` - Path to the file to upload
///
/// # Returns
///
/// The IPFS hash of the file or an error
pub fn upload_file<P: AsRef<Path>>(path: P) -> Result<String> {
let path = path.as_ref();
// Check if the file exists
if !path.exists() {
return Err(WebBuilderError::MissingFile(path.to_path_buf()));
}
// Create a tokio runtime
let rt = Runtime::new()
.map_err(|e| WebBuilderError::Other(format!("Failed to create tokio runtime: {}", e)))?;
// Upload the file to IPFS
let client = IpfsClient::default();
let ipfs_hash = rt.block_on(async {
// Open the file directly - this implements Read trait
let file = File::open(path).map_err(|e| WebBuilderError::IoError(e))?;
client
.add(file)
.await
.map_err(|e| WebBuilderError::IpfsError(format!("Failed to upload to IPFS: {}", e)))
.map(|res| res.hash)
})?;
Ok(ipfs_hash)
}
/// Calculate the Blake3 hash of a file
///
/// # Arguments
///
/// * `path` - Path to the file to hash
///
/// # Returns
///
/// The Blake3 hash of the file or an error
pub fn calculate_blake_hash<P: AsRef<Path>>(path: P) -> Result<String> {
let path = path.as_ref();
// Check if the file exists
if !path.exists() {
return Err(WebBuilderError::MissingFile(path.to_path_buf()));
}
// Read the file
let content = std::fs::read(path).map_err(|e| WebBuilderError::IoError(e))?;
// Calculate the hash
let hash = blake3::hash(&content);
let hash_hex = hash.to_hex().to_string();
Ok(format!("blake3-{}", hash_hex))
}

64
webbuilder/src/ipfs_test.rs
View File

@ -1,64 +0,0 @@
#[cfg(test)]
mod tests {
use crate::error::WebBuilderError;
use crate::ipfs::{calculate_blake_hash, upload_file};
use std::fs;
use tempfile::TempDir;
#[test]
fn test_upload_file_missing_file() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("nonexistent.txt");
let result = upload_file(&file_path);
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
WebBuilderError::MissingFile(_)
));
}
#[test]
fn test_calculate_blake_hash() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test.txt");
fs::write(&file_path, "test content").unwrap();
let result = calculate_blake_hash(&file_path).unwrap();
// The hash should start with "blake3-"
assert!(result.starts_with("blake3-"));
// The hash should be 64 characters long after the prefix
assert_eq!(result.len(), "blake3-".len() + 64);
// The hash should be the same for the same content
let file_path2 = temp_dir.path().join("test2.txt");
fs::write(&file_path2, "test content").unwrap();
let result2 = calculate_blake_hash(&file_path2).unwrap();
assert_eq!(result, result2);
// The hash should be different for different content
let file_path3 = temp_dir.path().join("test3.txt");
fs::write(&file_path3, "different content").unwrap();
let result3 = calculate_blake_hash(&file_path3).unwrap();
assert_ne!(result, result3);
}
#[test]
fn test_calculate_blake_hash_missing_file() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("nonexistent.txt");
let result = calculate_blake_hash(&file_path);
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
WebBuilderError::MissingFile(_)
));
}
}

59
webbuilder/src/lib.rs
View File

@ -1,59 +0,0 @@
//! WebBuilder is a library for building websites from hjson configuration files and markdown content.
//!
//! It uses the DocTree library to process markdown content and includes, and exports the result
//! to a webmeta.json file that can be used by a browser-based website generator.
pub mod builder;
pub mod config;
pub mod error;
pub mod git;
pub mod ipfs;
pub mod parser;
#[cfg(test)]
mod config_test;
#[cfg(test)]
mod error_test;
#[cfg(test)]
mod git_test;
#[cfg(test)]
mod ipfs_test;
#[cfg(test)]
mod parser_test;
pub use builder::WebBuilder;
pub use config::SiteConfig;
pub use error::{Result, WebBuilderError};
pub use parser::{ParsingStrategy, parse_site_config_with_strategy as parse_site_config, parse_site_config_recommended, parse_site_config_auto};
/// Create a new WebBuilder instance from a directory containing configuration files.
///
/// # Arguments
///
/// * `path` - Path to the directory containing configuration files
///
/// # Returns
///
/// A new WebBuilder instance or an error
pub fn from_directory<P: AsRef<std::path::Path>>(path: P) -> Result<WebBuilder> {
WebBuilder::from_directory(path)
}
/// Create a new WebBuilder instance from a directory containing configuration files,
/// using the specified parsing strategy.
///
/// # Arguments
///
/// * `path` - Path to the directory containing configuration files
/// * `strategy` - Parsing strategy to use
///
/// # Returns
///
/// A new WebBuilder instance or an error
pub fn from_directory_with_strategy<P: AsRef<std::path::Path>>(
path: P,
strategy: ParsingStrategy,
) -> Result<WebBuilder> {
let config = parser::parse_site_config_with_strategy(path, strategy)?;
Ok(WebBuilder { config })
}

88
webbuilder/src/main.rs
View File

@ -1,88 +0,0 @@
use clap::{Parser, Subcommand};
use std::path::PathBuf;
use webbuilder::{from_directory, Result};
#[derive(Parser)]
#[command(author, version, about, long_about = None)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Build a website from hjson configuration files
Build {
/// Path to the directory containing hjson configuration files
#[arg(short, long)]
path: PathBuf,
/// Output directory for the webmeta.json file
#[arg(short, long)]
output: Option<PathBuf>,
/// Whether to upload the webmeta.json file to IPFS
#[arg(short, long)]
upload: bool,
},
}
fn main() -> Result<()> {
// Initialize logger
env_logger::init();
// Parse command line arguments
let cli = Cli::parse();
// Handle commands
match &cli.command {
Commands::Build {
path,
output,
upload,
} => {
// Create a WebBuilder instance
let webbuilder = from_directory(path)?;
// Print the parsed configuration
println!("Parsed site configuration:");
println!(" Name: {}", webbuilder.config.name);
println!(" Title: {}", webbuilder.config.title);
println!(" Description: {:?}", webbuilder.config.description);
println!(" URL: {:?}", webbuilder.config.url);
println!(
" Collections: {} items",
webbuilder.config.collections.len()
);
for (i, collection) in webbuilder.config.collections.iter().enumerate() {
println!(
" Collection {}: {:?} - {:?}",
i, collection.name, collection.url
);
}
println!(" Pages: {} items", webbuilder.config.pages.len());
// Build the website
let webmeta = webbuilder.build()?;
// Save the webmeta.json file
let output_path = output
.clone()
.unwrap_or_else(|| PathBuf::from("webmeta.json"));
webmeta.save(&output_path)?;
// Upload to IPFS if requested
if *upload {
let ipfs_hash = webbuilder.upload_to_ipfs(&output_path)?;
println!("Uploaded to IPFS: {}", ipfs_hash);
}
println!("Website built successfully!");
println!("Output: {:?}", output_path);
}
}
Ok(())
}

517
webbuilder/src/parser.rs
View File

@ -1,517 +0,0 @@
use std::fs;
use std::path::Path;
use deser_hjson::from_str;
use serde::de::DeserializeOwned;
use serde_json::{self, Value};
use crate::config::{CollectionConfig, FooterConfig, HeaderConfig, PageConfig, SiteConfig};
use crate::error::{Result, WebBuilderError};
/// Parsing strategy to use
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ParsingStrategy {
/// Use the deser-hjson library (recommended)
Hjson,
/// Use a simple line-by-line parser (legacy)
Simple,
/// Auto-detect the best parser to use
Auto,
}
/// Parse a file into a struct using the specified strategy
///
/// # Arguments
///
/// * `path` - Path to the file to parse
/// * `strategy` - Parsing strategy to use
///
/// # Returns
///
/// The parsed struct or an error
pub fn parse_file<T, P>(path: P, strategy: ParsingStrategy) -> Result<T>
where
T: DeserializeOwned,
P: AsRef<Path>,
{
let path = path.as_ref();
// Check if the file exists
if !path.exists() {
return Err(WebBuilderError::MissingFile(path.to_path_buf()));
}
// Read the file
let content = fs::read_to_string(path).map_err(|e| WebBuilderError::IoError(e))?;
match strategy {
ParsingStrategy::Hjson => {
// Use the deser-hjson library
from_str(&content).map_err(|e| WebBuilderError::HjsonError(format!("Error parsing {:?}: {}", path, e)))
}
ParsingStrategy::Simple => {
// Use the simple parser - for this we need to handle the file reading ourselves
// since the original parse_hjson function does that internally
let path_ref: &Path = path.as_ref();
// Check if the file exists
if !path_ref.exists() {
return Err(WebBuilderError::MissingFile(path_ref.to_path_buf()));
}
// Read the file
let content = fs::read_to_string(path).map_err(|e| WebBuilderError::IoError(e))?;
// First try to parse as JSON
let json_result = serde_json::from_str::<T>(&content);
if json_result.is_ok() {
return Ok(json_result.unwrap());
}
// If that fails, try to convert hjson to json using a simple approach
let json_content = convert_hjson_to_json(&content)?;
// Parse the JSON
serde_json::from_str(&json_content)
.map_err(|e| WebBuilderError::HjsonError(format!("Error parsing {:?}: {}", path, e)))
}
ParsingStrategy::Auto => {
// Try the hjson parser first, fall back to simple if it fails
match from_str(&content) {
Ok(result) => Ok(result),
Err(e) => {
log::warn!("Hjson parser failed: {}, falling back to simple parser", e);
// Call the simple parser directly
let path_ref: &Path = path.as_ref();
// Check if the file exists
if !path_ref.exists() {
return Err(WebBuilderError::MissingFile(path_ref.to_path_buf()));
}
// Read the file
let content = fs::read_to_string(path).map_err(|e| WebBuilderError::IoError(e))?;
// First try to parse as JSON
let json_result = serde_json::from_str::<T>(&content);
if json_result.is_ok() {
return Ok(json_result.unwrap());
}
// If that fails, try to convert hjson to json using a simple approach
let json_content = convert_hjson_to_json(&content)?;
// Parse the JSON
serde_json::from_str(&json_content)
.map_err(|e| WebBuilderError::HjsonError(format!("Error parsing {:?}: {}", path, e)))
}
}
}
}
}
/// Parse a hjson file into a struct using the simple parser
///
/// # Arguments
///
/// * `path` - Path to the hjson file
///
/// # Returns
///
/// The parsed struct or an error
pub fn parse_hjson<T, P>(path: P) -> Result<T>
where
T: DeserializeOwned,
P: AsRef<Path>,
{
let path = path.as_ref();
// Check if the file exists
if !path.exists() {
return Err(WebBuilderError::MissingFile(path.to_path_buf()));
}
// Read the file
let content = fs::read_to_string(path).map_err(|e| WebBuilderError::IoError(e))?;
// First try to parse as JSON
let json_result = serde_json::from_str::<T>(&content);
if json_result.is_ok() {
return Ok(json_result.unwrap());
}
// If that fails, try to convert hjson to json using a simple approach
let json_content = convert_hjson_to_json(&content)?;
// Parse the JSON
serde_json::from_str(&json_content)
.map_err(|e| WebBuilderError::HjsonError(format!("Error parsing {:?}: {}", path, e)))
}
/// Convert hjson to json using a simple approach
///
/// # Arguments
///
/// * `hjson` - The hjson content
///
/// # Returns
///
/// The json content or an error
fn convert_hjson_to_json(hjson: &str) -> Result<String> {
// Remove comments
let mut json = String::new();
let mut lines = hjson.lines();
while let Some(line) = lines.next() {
let trimmed = line.trim();
// Skip empty lines
if trimmed.is_empty() {
continue;
}
// Skip comment lines
if trimmed.starts_with('#') {
continue;
}
// Handle key-value pairs
if let Some(pos) = trimmed.find(':') {
let key = trimmed[..pos].trim();
let value = trimmed[pos + 1..].trim();
// Add quotes to keys
json.push_str(&format!("\"{}\":", key));
// Add value
if value.is_empty() {
// If value is empty, it might be an object or array start
if lines
.clone()
.next()
.map_or(false, |l| l.trim().starts_with('{'))
{
json.push_str(" {");
} else if lines
.clone()
.next()
.map_or(false, |l| l.trim().starts_with('['))
{
json.push_str(" [");
} else {
json.push_str(" null");
}
} else {
// Add quotes to string values
if value.starts_with('"')
|| value.starts_with('[')
|| value.starts_with('{')
|| value == "true"
|| value == "false"
|| value == "null"
|| value.parse::<f64>().is_ok()
{
json.push_str(&format!(" {}", value));
} else {
json.push_str(&format!(" \"{}\"", value.replace('"', "\\\"")));
}
}
json.push_str(",\n");
} else if trimmed == "{" || trimmed == "[" {
json.push_str(trimmed);
json.push_str("\n");
} else if trimmed == "}" || trimmed == "]" {
// Remove trailing comma if present
if json.ends_with(",\n") {
json.pop();
json.pop();
json.push_str("\n");
}
json.push_str(trimmed);
json.push_str(",\n");
} else {
// Just copy the line
json.push_str(trimmed);
json.push_str("\n");
}
}
// Remove trailing comma if present
if json.ends_with(",\n") {
json.pop();
json.pop();
json.push_str("\n");
}
// Wrap in object if not already
if !json.trim().starts_with('{') {
json = format!("{{\n{}\n}}", json);
}
Ok(json)
}
/// Parse site configuration from a directory
///
/// # Arguments
///
/// * `path` - Path to the directory containing hjson configuration files
///
/// # Returns
///
/// The parsed site configuration or an error
pub fn parse_site_config<P: AsRef<Path>>(path: P) -> Result<SiteConfig> {
let path = path.as_ref();
// Check if the directory exists
if !path.exists() {
return Err(WebBuilderError::MissingDirectory(path.to_path_buf()));
}
// Check if the directory is a directory
if !path.is_dir() {
return Err(WebBuilderError::InvalidConfiguration(format!(
"{:?} is not a directory",
path
)));
}
// Parse main.hjson
let main_path = path.join("main.hjson");
let main_config: serde_json::Value = parse_hjson(main_path)?;
// Parse header.hjson
let header_path = path.join("header.hjson");
let header_config: Option<HeaderConfig> = if header_path.exists() {
Some(parse_hjson(header_path)?)
} else {
None
};
// Parse footer.hjson
let footer_path = path.join("footer.hjson");
let footer_config: Option<FooterConfig> = if footer_path.exists() {
Some(parse_hjson(footer_path)?)
} else {
None
};
// Parse collection.hjson
let collection_path = path.join("collection.hjson");
let collection_configs: Vec<CollectionConfig> = if collection_path.exists() {
parse_hjson(collection_path)?
} else {
Vec::new()
};
// Parse pages directory
let pages_path = path.join("pages");
let mut page_configs: Vec<PageConfig> = Vec::new();
if pages_path.exists() && pages_path.is_dir() {
for entry in fs::read_dir(pages_path)? {
let entry = entry?;
let entry_path = entry.path();
if entry_path.is_file() && entry_path.extension().map_or(false, |ext| ext == "hjson") {
let page_config: Vec<PageConfig> = parse_hjson(&entry_path)?;
page_configs.extend(page_config);
}
}
}
// Parse keywords from main.hjson
let keywords = if let Some(keywords_value) = main_config.get("keywords") {
if keywords_value.is_array() {
let mut keywords_vec = Vec::new();
for keyword in keywords_value.as_array().unwrap() {
if let Some(keyword_str) = keyword.as_str() {
keywords_vec.push(keyword_str.to_string());
}
}
Some(keywords_vec)
} else if let Some(keywords_str) = keywords_value.as_str() {
// Handle comma-separated keywords
Some(
keywords_str
.split(',')
.map(|s| s.trim().to_string())
.collect(),
)
} else {
None
}
} else {
None
};
// Create site configuration
let site_config = SiteConfig {
name: main_config["name"]
.as_str()
.unwrap_or("default")
.to_string(),
title: main_config["title"].as_str().unwrap_or("").to_string(),
description: main_config["description"].as_str().map(|s| s.to_string()),
keywords,
url: main_config["url"].as_str().map(|s| s.to_string()),
favicon: main_config["favicon"].as_str().map(|s| s.to_string()),
header: header_config,
footer: footer_config,
collections: collection_configs,
pages: page_configs,
base_path: path.to_path_buf(),
};
Ok(site_config)
}
/// Parse site configuration from a directory using the specified strategy
///
/// # Arguments
///
/// * `path` - Path to the directory containing configuration files
/// * `strategy` - Parsing strategy to use
///
/// # Returns
///
/// The parsed site configuration or an error
pub fn parse_site_config_with_strategy<P: AsRef<Path>>(path: P, strategy: ParsingStrategy) -> Result<SiteConfig> {
let path = path.as_ref();
// Check if the directory exists
if !path.exists() {
return Err(WebBuilderError::MissingDirectory(path.to_path_buf()));
}
// Check if the directory is a directory
if !path.is_dir() {
return Err(WebBuilderError::InvalidConfiguration(format!(
"{:?} is not a directory",
path
)));
}
// Create a basic site configuration
let mut site_config = SiteConfig {
name: "default".to_string(),
title: "".to_string(),
description: None,
keywords: None,
url: None,
favicon: None,
header: None,
footer: None,
collections: Vec::new(),
pages: Vec::new(),
base_path: path.to_path_buf(),
};
// Parse main.hjson
let main_path = path.join("main.hjson");
if main_path.exists() {
let main_config: Value = parse_file(main_path, strategy)?;
// Extract values from main.hjson
if let Some(name) = main_config.get("name").and_then(|v| v.as_str()) {
site_config.name = name.to_string();
}
if let Some(title) = main_config.get("title").and_then(|v| v.as_str()) {
site_config.title = title.to_string();
}
if let Some(description) = main_config.get("description").and_then(|v| v.as_str()) {
site_config.description = Some(description.to_string());
}
if let Some(url) = main_config.get("url").and_then(|v| v.as_str()) {
site_config.url = Some(url.to_string());
}
if let Some(favicon) = main_config.get("favicon").and_then(|v| v.as_str()) {
site_config.favicon = Some(favicon.to_string());
}
if let Some(keywords) = main_config.get("keywords").and_then(|v| v.as_array()) {
let keywords_vec: Vec<String> = keywords
.iter()
.filter_map(|k| k.as_str().map(|s| s.to_string()))
.collect();
if !keywords_vec.is_empty() {
site_config.keywords = Some(keywords_vec);
}
}
}
// Parse header.hjson
let header_path = path.join("header.hjson");
if header_path.exists() {
site_config.header = Some(parse_file(header_path, strategy)?);
}
// Parse footer.hjson
let footer_path = path.join("footer.hjson");
if footer_path.exists() {
site_config.footer = Some(parse_file(footer_path, strategy)?);
}
// Parse collection.hjson
let collection_path = path.join("collection.hjson");
if collection_path.exists() {
let collection_array: Vec<CollectionConfig> = parse_file(collection_path, strategy)?;
// Process each collection
for mut collection in collection_array {
// Convert web interface URL to Git URL if needed
if let Some(url) = &collection.url {
if url.contains("/src/branch/") {
// This is a web interface URL, convert it to a Git URL
let parts: Vec<&str> = url.split("/src/branch/").collect();
if parts.len() == 2 {
collection.url = Some(format!("{}.git", parts[0]));
}
}
}
site_config.collections.push(collection);
}
}
// Parse pages directory
let pages_path = path.join("pages");
if pages_path.exists() && pages_path.is_dir() {
for entry in fs::read_dir(pages_path)? {
let entry = entry?;
let entry_path = entry.path();
if entry_path.is_file() && entry_path.extension().map_or(false, |ext| ext == "hjson") {
let pages_array: Vec<PageConfig> = parse_file(&entry_path, strategy)?;
site_config.pages.extend(pages_array);
}
}
}
Ok(site_config)
}
/// Parse site configuration from a directory using the recommended strategy (Hjson)
///
/// # Arguments
///
/// * `path` - Path to the directory containing configuration files
///
/// # Returns
///
/// The parsed site configuration or an error
pub fn parse_site_config_recommended<P: AsRef<Path>>(path: P) -> Result<SiteConfig> {
parse_site_config_with_strategy(path, ParsingStrategy::Hjson)
}
/// Parse site configuration from a directory using the auto-detect strategy
///
/// # Arguments
///
/// * `path` - Path to the directory containing configuration files
///
/// # Returns
///
/// The parsed site configuration or an error
pub fn parse_site_config_auto<P: AsRef<Path>>(path: P) -> Result<SiteConfig> {
parse_site_config_with_strategy(path, ParsingStrategy::Auto)
}

267
webbuilder/src/parser_test.rs
View File

@ -1,267 +0,0 @@
#[cfg(test)]
mod tests {
use crate::error::WebBuilderError;
use crate::parser::{parse_site_config_with_strategy, ParsingStrategy};
use std::fs;
use std::path::PathBuf;
use tempfile::TempDir;
fn create_test_site(temp_dir: &TempDir) -> PathBuf {
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create main.hjson
let main_hjson = r#"{
# Main configuration
"name": "test",
"title": "Test Site",
"description": "A test site",
"url": "https://example.com",
"favicon": "favicon.ico",
"keywords": [
"demo",
"test",
"example"
]
}"#;
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
// Create header.hjson
let header_hjson = r#"{
# Header configuration
"title": "Test Site",
"logo": {
"src": "logo.png",
"alt": "Logo"
},
"menu": [
{
"label": "Home",
"link": "/"
},
{
"label": "About",
"link": "/about"
}
]
}"#;
fs::write(site_dir.join("header.hjson"), header_hjson).unwrap();
// Create collection.hjson
let collection_hjson = r#"[
{
# First collection
"name": "test",
"url": "https://git.ourworld.tf/tfgrid/home.git",
"description": "A test collection",
"scan": true
},
{
# Second collection
"name": "test2",
"url": "https://git.example.com/src/branch/main/test2",
"description": "Another test collection"
}
]"#;
fs::write(site_dir.join("collection.hjson"), collection_hjson).unwrap();
// Create pages directory
let pages_dir = site_dir.join("pages");
fs::create_dir(&pages_dir).unwrap();
// Create pages/pages.hjson
let pages_hjson = r#"[
{
# Home page
"name": "home",
"title": "Home",
"description": "Home page",
"navpath": "/",
"collection": "test",
"draft": false
},
{
# About page
"name": "about",
"title": "About",
"description": "About page",
"navpath": "/about",
"collection": "test"
}
]"#;
fs::write(pages_dir.join("pages.hjson"), pages_hjson).unwrap();
site_dir
}
#[test]
fn test_parse_site_config_hjson() {
let temp_dir = TempDir::new().unwrap();
let site_dir = create_test_site(&temp_dir);
let config = parse_site_config_with_strategy(&site_dir, ParsingStrategy::Hjson).unwrap();
// Check basic site info
assert_eq!(config.name, "test");
assert_eq!(config.title, "Test Site");
assert_eq!(config.description, Some("A test site".to_string()));
assert_eq!(config.url, Some("https://example.com".to_string()));
assert_eq!(config.favicon, Some("favicon.ico".to_string()));
assert_eq!(
config.keywords,
Some(vec![
"demo".to_string(),
"test".to_string(),
"example".to_string()
])
);
// Check header
assert!(config.header.is_some());
let header = config.header.as_ref().unwrap();
assert_eq!(header.title, Some("Test Site".to_string()));
assert!(header.logo.is_some());
let logo = header.logo.as_ref().unwrap();
assert_eq!(logo.src, "logo.png");
assert_eq!(logo.alt, Some("Logo".to_string()));
// Check collections
assert_eq!(config.collections.len(), 2);
// First collection
assert_eq!(config.collections[0].name, Some("test".to_string()));
assert_eq!(
config.collections[0].url,
Some("https://git.ourworld.tf/tfgrid/home.git".to_string())
);
assert_eq!(
config.collections[0].description,
Some("A test collection".to_string())
);
assert_eq!(config.collections[0].scan, Some(true));
// Second collection (with URL conversion)
assert_eq!(config.collections[1].name, Some("test2".to_string()));
assert_eq!(
config.collections[1].url,
Some("https://git.example.com.git".to_string())
);
assert_eq!(
config.collections[1].description,
Some("Another test collection".to_string())
);
assert_eq!(config.collections[1].scan, None);
// Check pages
assert_eq!(config.pages.len(), 2);
// First page
assert_eq!(config.pages[0].name, "home");
assert_eq!(config.pages[0].title, "Home");
assert_eq!(config.pages[0].description, Some("Home page".to_string()));
assert_eq!(config.pages[0].navpath, "/");
assert_eq!(config.pages[0].collection, "test");
assert_eq!(config.pages[0].draft, Some(false));
// Second page
assert_eq!(config.pages[1].name, "about");
assert_eq!(config.pages[1].title, "About");
assert_eq!(config.pages[1].description, Some("About page".to_string()));
assert_eq!(config.pages[1].navpath, "/about");
assert_eq!(config.pages[1].collection, "test");
assert_eq!(config.pages[1].draft, None);
}
#[test]
fn test_parse_site_config_auto() {
let temp_dir = TempDir::new().unwrap();
let site_dir = create_test_site(&temp_dir);
let config = parse_site_config_with_strategy(&site_dir, ParsingStrategy::Auto).unwrap();
// Basic checks to ensure it worked
assert_eq!(config.name, "test");
assert_eq!(config.title, "Test Site");
assert_eq!(config.collections.len(), 2);
assert_eq!(config.pages.len(), 2);
}
#[test]
fn test_parse_site_config_simple() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create main.hjson in a format that the simple parser can handle
let main_hjson = "name: test\ntitle: Test Site\ndescription: A test site";
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
let config = parse_site_config_with_strategy(&site_dir, ParsingStrategy::Simple).unwrap();
// Basic checks to ensure it worked
assert_eq!(config.name, "test");
assert_eq!(config.title, "Test Site");
assert_eq!(config.description, Some("A test site".to_string()));
}
#[test]
fn test_parse_site_config_missing_directory() {
let result = parse_site_config_with_strategy("/nonexistent/directory", ParsingStrategy::Hjson);
assert!(matches!(result, Err(WebBuilderError::MissingDirectory(_))));
}
#[test]
fn test_parse_site_config_not_a_directory() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("file.txt");
fs::write(&file_path, "not a directory").unwrap();
let result = parse_site_config_with_strategy(&file_path, ParsingStrategy::Hjson);
assert!(matches!(
result,
Err(WebBuilderError::InvalidConfiguration(_))
));
}
#[test]
fn test_parse_site_config_minimal() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create minimal main.hjson
let main_hjson = r#"{ "name": "minimal", "title": "Minimal Site" }"#;
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
let config = parse_site_config_with_strategy(&site_dir, ParsingStrategy::Hjson).unwrap();
assert_eq!(config.name, "minimal");
assert_eq!(config.title, "Minimal Site");
assert_eq!(config.description, None);
assert_eq!(config.url, None);
assert_eq!(config.favicon, None);
assert!(config.header.is_none());
assert!(config.footer.is_none());
assert!(config.collections.is_empty());
assert!(config.pages.is_empty());
}
#[test]
fn test_parse_site_config_empty() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
let config = parse_site_config_with_strategy(&site_dir, ParsingStrategy::Hjson).unwrap();
assert_eq!(config.name, "default");
assert_eq!(config.title, "");
assert_eq!(config.description, None);
assert_eq!(config.url, None);
assert_eq!(config.favicon, None);
assert!(config.header.is_none());
assert!(config.footer.is_none());
assert!(config.collections.is_empty());
assert!(config.pages.is_empty());
}
}