...

examples/web/.gitignore

@@ -1,2 +1,8 @@
 build
-docusaurus_example
+docusaurus_example
+mdbook_example
+markdown_example
+markdown_example0
+doctree_example
+tree_scan
+*.dSYM

examples/web/doctree/doctree_client.vsh

@@ -0,0 +1,128 @@
+#!/usr/bin/env -S v -n -w -cg -gc none  -cc tcc -d use_openssl -enable-globals run
+
+
+import freeflowuniverse.herolib.data.doctree
+import freeflowuniverse.herolib.web.doctreeclient
+import os
+
+println('This example demonstrates how to use the DocTreeClient to interact with doctree data.')
+println('First, ensure doctree data is populated in Redis. This step is usually done once.')
+
+// Populate Redis with doctree data (if not already done)
+// This example uses a public Git repository for demonstration.
+// In a real scenario, you might use your own documentation repository.
+mut tree := doctree.new(name: 'tfgrid_docs')! // Using a distinct name for the example
+tree.scan(
+	git_url:  'https://git.threefold.info/tfgrid/docs_tfgrid4/src/branch/main/collections'
+	git_pull: false // Set to true to pull latest changes
+)!
+
+// Exporting data is optional, but useful for local access or debugging
+// tree.export(
+// 	destination:    '/tmp/doctree_example_export'
+// 	reset:          true
+// 	frontmatter_remove: true //remove frontmatter from markdown files
+// )!
+
+println('Doctree data setup complete.')
+
+// Create a DocTreeClient instance
+mut client := doctreeclient.new()!
+
+// List all available collections
+println('\n--- Listing Collections ---')
+collections := client.list_collections()!
+if collections.len == 0 {
+	println('No collections found. Please ensure doctree data is correctly populated.')
+	return
+}
+
+println('Found ${collections.len} collections:')
+for collection in collections {
+	println('- ${collection}')
+}
+
+// Use the first collection found for further demonstration
+collection_name := collections[0]
+println('\n--- Using collection: ${collection_name} ---')
+
+// List pages within the selected collection
+println('\n--- Listing Pages in ${collection_name} ---')
+pages := client.list_pages(collection_name)!
+if pages.len == 0 {
+	println('No pages found in collection "${collection_name}".')
+	return
+}
+
+println('Found ${pages.len} pages:')
+for page in pages {
+	println('- ${page}')
+}
+
+// Get content of the first page
+page_to_get := pages[0]
+println('\n--- Getting content for page: ${page_to_get} ---')
+page_content := client.get_page_content(collection_name, page_to_get)!
+println('Content of "${page_to_get}" (first 200 chars):\n${page_content[..200]}...')
+
+// Check if a specific page exists
+println('\n--- Checking page existence ---')
+exists := client.page_exists(collection_name, page_to_get)
+println('Page "${page_to_get}" exists: ${exists}')
+
+non_existent_page := 'non_existent_page_123'
+exists_non_existent := client.page_exists(collection_name, non_existent_page)
+println('Page "${non_existent_page}" exists: ${exists_non_existent}')
+
+
+// Step 7: List images in the collection
+println('\n7. Listing images:')
+images := client.list_images(collection_name)!
+println('Found ${images.len} images: ${images}')
+
+// Step 8: Get image path
+if images.len > 0 {
+	image_name := images[0]
+	println('\n8. Getting path of image: ${image_name}')
+
+	// Check if image exists
+	exists2 := client.image_exists(collection_name, image_name)
+	println('Image exists: ${exists2}')
+
+	// Get image path
+	image_path := client.get_image_path(collection_name, image_name)!
+	println('Image path: ${image_path}')
+}
+
+// Step 9: List files in the collection
+println('\n9. Listing files:')
+files := client.list_files(collection_name)!
+println('Found ${files.len} files: ${files}')
+
+// Step 10: Get file path
+if files.len > 0 {
+	file_name := files[0]
+	println('\n10. Getting path of file: ${file_name}')
+
+	// Check if file exists
+	exists3 := client.file_exists(collection_name, file_name)
+	println('File exists: ${exists3}')
+
+	// Get file path
+	file_path := client.get_file_path(collection_name, file_name)!
+	println('File path: ${file_path}')
+}
+
+// Step 11: Error handling example
+println('\n11. Error handling example:')
+println('Trying to access a non-existent page...')
+
+non_existent_page2 := 'non_existent_page_2'
+content := client.get_page_content(collection_name, non_existent_page2) or {
+	println('Error caught: ${err}')
+	'Error content'
+}
+
+
+println('\nExample completed successfully!')
+

examples/web/doctreeclient_example.vsh

@@ -1,124 +0,0 @@
-#!/usr/bin/env -S v -n -w -cg -gc none -cc tcc -d use_openssl -enable-globals run
-
-import freeflowuniverse.herolib.web.doctreeclient
-import freeflowuniverse.herolib.data.doctree
-import os
-
-println('DocTreeClient Example')
-println('=====================')
-
-// Step 1: First, populate Redis with doctree data
-println('\n1. Setting up doctree data in Redis...')
-
-tree.scan(
-	git_url:  'https://git.threefold.info/tfgrid/docs_tfgrid4/src/branch/main/collections'
-	git_pull: false
-)!
-
-tree.export(
-	destination:    '/tmp/mdexport'
-	reset:          true
-	exclude_errors: false
-)!
-
-println('Doctree data populated in Redis')
-
-// Step 2: Create a DocTreeClient instance
-println('\n2. Creating DocTreeClient...')
-mut client := doctreeclient.new()!
-println('DocTreeClient created successfully')
-
-// Step 3: List all collections
-println('\n3. Listing collections:')
-collections := client.list_collections()!
-println('Found ${collections.len} collections: ${collections}')
-
-if collections.len == 0 {
-	println('No collections found. Example cannot continue.')
-	return
-}
-
-// Step 4: Use the example_docs collection
-collection_name := 'example_docs'
-println('\n4. Using collection: ${collection_name}')
-
-// Step 5: List pages in the collection
-println('\n5. Listing pages:')
-pages := client.list_pages(collection_name)!
-println('Found ${pages.len} pages: ${pages}')
-
-// Step 6: Get content of a page
-if pages.len > 0 {
-	page_name := 'introduction'
-	println('\n6. Getting content of page: ${page_name}')
-
-	// Check if page exists
-	exists := client.page_exists(collection_name, page_name)
-	println('Page exists: ${exists}')
-
-	// Get page path
-	page_path := client.get_page_path(collection_name, page_name)!
-	println('Page path: ${page_path}')
-
-	// Get page content
-	content := client.get_page_content(collection_name, page_name)!
-	println('Page content:')
-	println('---')
-	println(content)
-	println('---')
-}
-
-// Step 7: List images in the collection
-println('\n7. Listing images:')
-images := client.list_images(collection_name)!
-println('Found ${images.len} images: ${images}')
-
-// Step 8: Get image path
-if images.len > 0 {
-	image_name := images[0]
-	println('\n8. Getting path of image: ${image_name}')
-
-	// Check if image exists
-	exists := client.image_exists(collection_name, image_name)
-	println('Image exists: ${exists}')
-
-	// Get image path
-	image_path := client.get_image_path(collection_name, image_name)!
-	println('Image path: ${image_path}')
-}
-
-// Step 9: List files in the collection
-println('\n9. Listing files:')
-files := client.list_files(collection_name)!
-println('Found ${files.len} files: ${files}')
-
-// Step 10: Get file path
-if files.len > 0 {
-	file_name := files[0]
-	println('\n10. Getting path of file: ${file_name}')
-
-	// Check if file exists
-	exists := client.file_exists(collection_name, file_name)
-	println('File exists: ${exists}')
-
-	// Get file path
-	file_path := client.get_file_path(collection_name, file_name)!
-	println('File path: ${file_path}')
-}
-
-// Step 11: Error handling example
-println('\n11. Error handling example:')
-println('Trying to access a non-existent page...')
-
-non_existent_page := 'non_existent_page'
-content := client.get_page_content(collection_name, non_existent_page) or {
-	println('Error caught: ${err}')
-	'Error content'
-}
-
-// Step 12: Clean up
-println('\n12. Cleaning up...')
-os.rmdir_all(example_dir) or { println('Failed to remove example directory: ${err}') }
-os.rmdir_all(export_dir) or { println('Failed to remove export directory: ${err}') }
-
-println('\nExample completed successfully!')

examples/web/mdbook_markdown/.gitignore

@@ -1,6 +0,0 @@
-mdbook_example
-markdown_example
-markdown_example0
-doctree_example
-tree_scan
-*.dSYM

lib/data/doctree/README.md

@@ -1,9 +1,5 @@
 # Doctree Module
 
-The `doctree` module is a V language library designed for scanning, processing, and exporting collections of documents. It provides a structured way to manage document-based content, making it suitable for generating documentation, building static websites, or processing any content organized into collections.
-
-## Purpose
-
 The primary goal of this module is to transform structured document collections into a format suitable for various outputs. It handles the complexities of finding collections, loading their content, processing includes, definitions, and macros, and exporting the final result while managing assets like images and files.
 
 ## Key Concepts
@@ -17,7 +13,7 @@ The primary goal of this module is to transform structured document collections
 The typical workflow involves creating a `Tree`, scanning for collections, and then exporting the processed content.å
 
 1.  **Create Tree:** Initialize a `doctree.Tree` instance using `doctree.new()`.
-2.  **Scan:** Use the `tree.scan()` or `tree.scan_concurrent()` method, providing a path to a directory or a Git repository URL. The scanner recursively looks for directories containing a `.collection` file.
+2.  **Scan:** Use the `tree.scan()`  method, providing a path to a directory or a Git repository URL. The scanner recursively looks for directories containing a `.collection` file.
 3.  **Load Content:** For each identified collection, the module loads its content, including markdown pages, images, and other files.
 4.  **Process Content:** The loaded content is processed. This includes handling definitions, includes (content from other files), and macros (dynamic content generation or transformation).
 5.  **Generate Output Paths:** The module determines the final paths for all processed files and assets in the destination directory.

lib/web/docusaurus/dsite_generate.v

@@ -7,6 +7,8 @@ import os
 import freeflowuniverse.herolib.osal
 import freeflowuniverse.herolib.ui.console
 import freeflowuniverse.herolib.core.texttools.regext
+import freeflowuniverse.herolib.data.doctree
+
 
 pub fn (mut site DocSite) generate() ! {
 	console.print_header(' site generate: ${site.name} on ${site.factory.path_build.path}')
@@ -53,6 +55,9 @@ pub fn (mut site DocSite) generate() ! {
 
 pub fn (mut site DocSite) download_collections() ! {
 
+	//this means we need to do doctree version
+	mut tree := doctree.new(name: 'site_${site.name}')!
+
 	mut gs := gittools.new()!
 	for item in site.siteconfig.import_collections {
 		mypath := gs.get_path(
@@ -61,7 +66,36 @@ pub fn (mut site DocSite) download_collections() ! {
 			url:   item.url
 		)!
 		mut mypatho := pathlib.get(mypath)
-		mypatho.copy(dest: '${site.factory.path_build.path}/docs/${item.dest}', delete: true)!
+
+		if item.frontmatter{
+			//if frontmatter specified then no need to do as collections just copy to destination
+			mypatho.copy(dest: '${site.factory.path_build.path}/docs/${item.dest}', delete: true)!
+		}else{
+			tree.add(
+				path: mypath
+				name: item.name
+			)!
+		}
+	}	
+
+	//now export all collections
+	tree.export(
+		destination:    '${site.factory.path_build.path}/collections'
+		reset:          true
+		exclude_errors: false
+	)!
+	for item in site.siteconfig.import_collections {
+		//if dest specified them we consider source to have the docusaurus parts
+		if item.dest!=""{
+			mypatho.copy(dest: '${site.factory.path_build.path}/docs/${item.dest}', delete: true)!
+			continue
+		}
+		
+		tree.add(
+			path: mypath
+			name: item.name
+		)!
+
 		// println(item)
 		//replace: {'NAME': 'MyName', 'URGENCY': 'red'}
 		mut ri := regext.regex_instructions_new()
@@ -71,7 +105,14 @@ pub fn (mut site DocSite) download_collections() ! {
 		// println(ri)
 		ri.replace_in_dir(path:"${site.factory.path_build.path}/docs/${item.dest}",extensions:["md"])!
 
+		if item.dest:=""{
+			mypatho.copy(dest: '${site.factory.path_build.path}/docs/${item.dest}', delete: true)!
+		}else{
+			mypatho.copy(dest: , delete: true)!
+		}
+
 		
-	}	
+	}		
+
 
 }

lib/web/siteconfig/config.v

@@ -88,9 +88,11 @@ pub mut:
 
 pub struct CollectionsImport {
 pub mut:
+	name string //will normally be empty
 	url     string // http git url can be to specific path
 	path    string
 	dest    string            // location in the docs folder of the place where we will build docusaurus
 	replace map[string]string // will replace ${NAME} in the imported content
 	visible bool = true
+	frontmatter bool //if frontmatter is part of the document
 }

lib/web/siteconfig/play.v

@@ -135,6 +135,8 @@ fn play_collections(mut plbook PlayBook, mut config SiteConfig) ! {
 			}
 		}
 		mut import_ := CollectionsImport{
+			name:        p.get_default('name','')!
+			frontmatter:        p.get_default('frontmatter','')!			
 			url:     p.get('url')!
 			path:    p.get_default('path', '')!
 			dest:    p.get_default('dest', '')!

specs.md

@@ -0,0 +1,46 @@
+# Doctree Export Specification
+
+## Overview
+The `doctree` module in `lib/data/doctree` is responsible for processing and exporting documentation trees. This involves taking a structured representation of documentation (collections, pages, images, files) and writing it to a specified file system destination. Additionally, it leverages Redis to store metadata about the exported documentation, facilitating quick lookups and integration with other systems.
+
+## Key Components
+
+### `lib/data/doctree/export.v`
+This file defines the main `export` function for the `Tree` object. It orchestrates the overall export process:
+- Takes `TreeExportArgs` which includes parameters like `destination`, `reset` (to clear destination), `keep_structure`, `exclude_errors`, `toreplace` (for regex replacements), `concurrent` (for parallel processing), and `redis` (to control Redis metadata storage).
+- Processes definitions, includes, actions, and macros within the `Tree`.
+- Generates file paths for pages, images, and other files.
+- Iterates through `Collection` objects within the `Tree` and calls their respective `export` methods, passing down the `redis` flag.
+
+### `lib/data/doctree/collection/export.v`
+This file defines the `export` function for the `Collection` object. This is where the actual file system writing and Redis interaction for individual collections occur:
+- Takes `CollectionExportArgs` which includes `destination`, `file_paths`, `reset`, `keep_structure`, `exclude_errors`, `replacer`, and the `redis` flag.
+- Creates a `.collection` file in the destination directory with basic collection information.
+- **Redis Integration**:
+    - Obtains a Redis client using `base.context().redis()`.
+    - Stores the collection's destination path in Redis using `redis.hset('doctree:path', 'collection_name', 'destination_path')`.
+    - Calls `export_pages`, `export_files`, `export_images`, and `export_linked_pages` which all interact with Redis if the `redis` flag is true.
+- **`export_pages`**:
+    - Processes page links and handles not-found errors.
+    - Writes markdown content to the destination file system.
+    - Stores page metadata in Redis: `redis.hset('doctree:collection_name', 'page_name', 'page_file_name.md')`.
+- **`export_files` and `export_images`**:
+    - Copies files and images to the destination directory (e.g., `img/`).
+    - Stores file/image metadata in Redis: `redis.hset('doctree:collection_name', 'file_name', 'img/file_name.ext')`.
+- **`export_linked_pages`**:
+    - Gathers linked pages within the collection.
+    - Writes a `.linkedpages` file.
+    - Stores linked pages file metadata in Redis: `redis.hset('doctree:collection_name', 'linkedpages', 'linkedpages_file_name.md')`.
+
+## Link between Redis and Export
+
+The `doctree` export process uses Redis as a metadata store. When the `redis` flag is set to `true` (which is the default), the export functions populate Redis with key-value pairs that map collection names, page names, file names, and image names to their respective paths and file names within the exported documentation structure.
+
+This Redis integration serves as a quick lookup mechanism for other applications or services that might need to access or reference the exported documentation. Instead of traversing the file system, these services can query Redis to get the location of specific documentation elements.
+
+## Is Export Needed?
+
+Yes, the export functionality is crucial for making the processed `doctree` content available outside the internal `doctree` representation.
+
+- **File System Export**: The core purpose of the export is to write the documentation content (markdown files, images, other assets) to a specified directory. This is essential for serving the documentation via a web server, integrating with static site generators (like Docusaurus, as suggested by other files in the project), or simply providing a browsable version of the documentation.
+- **Redis Metadata**: While the file system export is fundamental, the Redis metadata storage is an important complementary feature. It provides an efficient way for other systems to programmatically discover and locate documentation assets. If there are downstream applications that rely on this Redis metadata for navigation, search, or content delivery, then the Redis part of the export is indeed needed. If no such applications exist or are planned, the `redis` flag can be set to `false` to skip this step, but the file system export itself remains necessary for external consumption of the documentation.