...

examples/data/encrypt_decrypt.vsh

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S v -n -w -gc none  -cc tcc -d use_openssl -enable-globals run
+#!/usr/bin/env -S v -n -w -gc none -cg  -cc tcc -d use_openssl -enable-globals run
 
 import freeflowuniverse.herolib.crypt.aes_symmetric { decrypt, encrypt }
 import freeflowuniverse.herolib.ui.console

examples/webtools/mdbook_markdown/doctree_export.vsh

@@ -10,13 +10,10 @@ mut tree := doctree.new(name: 'test')!
 // git_reset bool
 // git_root  string
 // git_pull  bool
-// load      bool = true // means we scan automatically the added collection
+tree.scan(
-for project in 'projectinca, legal, why'.split(',').map(it.trim_space()) {
+	git_url:  'https://git.ourworld.tf/tfgrid/docs_tfgrid4/src/branch/main/collections'
-	tree.scan(
+	git_pull: false
-		git_url:  'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/development/collections/${project}'
+)!
-		git_pull: false
-	)!
-}
 
 tree.export(
 	destination: '/tmp/mdexport'

lib/data/doctree/README.md

@@ -0,0 +1,86 @@
+# Doctree Module
+
+The `doctree` module is a V language library designed for scanning, processing, and exporting collections of documents. It provides a structured way to manage document-based content, making it suitable for generating documentation, building static websites, or processing any content organized into collections.
+
+## Purpose
+
+The primary goal of this module is to transform structured document collections into a format suitable for various outputs. It handles the complexities of finding collections, loading their content, processing includes, definitions, and macros, and exporting the final result while managing assets like images and files.
+
+## Key Concepts
+
+*   **Tree:** The central component (`doctree.Tree`) that holds one or more `Collection` instances. It orchestrates the scanning, processing, and exporting of all contained collections.
+*   **Collection:** A directory that is marked as a collection by the presence of a `.collection` file. A collection groups related documents (pages, images, files) and can have its own configuration defined within the `.collection` file.
+*   **.collection file:** A file placed in a directory to designate it as a collection. This file can optionally contain parameters (using the `paramsparser` format) such as a custom name for the collection.
+
+## How it Works (Workflow)
+
+The typical workflow involves creating a `Tree`, scanning for collections, and then exporting the processed content.å
+
+1.  **Create Tree:** Initialize a `doctree.Tree` instance using `doctree.new()`.
+2.  **Scan:** Use the `tree.scan()` or `tree.scan_concurrent()` method, providing a path to a directory or a Git repository URL. The scanner recursively looks for directories containing a `.collection` file.
+3.  **Load Content:** For each identified collection, the module loads its content, including markdown pages, images, and other files.
+4.  **Process Content:** The loaded content is processed. This includes handling definitions, includes (content from other files), and macros (dynamic content generation or transformation).
+5.  **Generate Output Paths:** The module determines the final paths for all processed files and assets in the destination directory.
+6.  **Export:** The `tree.export()` method writes the processed content and assets to the specified destination directory, maintaining the desired structure.
+
+## Usage (For Developers)
+
+Here's a basic example of how to use the `doctree` module in your V project:
+
+```v
+import freeflowuniverse.herolib.data.doctree
+// 1. Create a new Tree instance
+mut tree := doctree.new(name: 'my_documentation')!
+
+// 2. Scan a directory containing your collections
+// Replace './docs' with the actual path to your document collections
+tree.scan(path: './docs')!
+
+// use from URL
+//git_url   string
+//git_reset bool
+//git_pull  bool
+tree.scan(git_url: 'https://git.ourworld.tf/tfgrid/docs_tfgrid4/src/branch/main/collections')!
+
+// 3. Export the processed content to a destination directory
+// Replace './output' with your desired output path
+// if redis then the metadata will be put in redis
+tree.export(destination: './output', redis:true)!
+
+println('Documentation successfully exported to ./output')
+
+```
+
+## Structure of a Collection
+
+A collection is a directory containing a `.collection` file. Inside a collection directory, you would typically organize your content like this:
+
+```
+my_collection/
+├── .collection
+├── page1.md
+├── page2.md
+├── images/
+│   ├── image1.png
+│   └── image2.jpg
+└── files/
+    ├── document.pdf
+    └── data.csv
+```
+
+Markdown files (`.md`) are treated as pages. 
+
+## Redis Structure
+
+when using the export redis:true argument, which is default
+
+in redis we will find
+
+```bash
+#redis hsets:
+doctree:$collectionname $pagename   $rel_path_in_collection
+doctree:$collectionname $filename.$ext  $rel_path_in_collection
+doctree:meta            $collectionname     $collectionpath_on_disk
+```
+
+

lib/data/doctree/collection/error.v

@@ -1,6 +1,7 @@
 module collection
 
 import freeflowuniverse.herolib.core.pathlib { Path }
+import freeflowuniverse.herolib.core.base
 import freeflowuniverse.herolib.ui.console
 
 pub enum CollectionErrorCat {
@@ -52,8 +53,10 @@ pub fn (err ObjNotFound) msg() string {
 }
 
 // write errors.md in the collection, this allows us to see what the errors are
-pub fn (collection Collection) errors_report(dest_ string) ! {
+pub fn (collection Collection) errors_report(col_name string,dest_ string) ! {
 	// console.print_debug("====== errors report: ${dest_} : ${collection.errors.len}\n${collection.errors}")
+	mut context := base.context()!
+	mut redis := context.redis()!	
 	mut dest := pathlib.get_file(path: dest_, create: true)!
 	if collection.errors.len == 0 {
 		dest.delete()!
@@ -61,4 +64,5 @@ pub fn (collection Collection) errors_report(dest_ string) ! {
 	}
 	c := $tmpl('template/errors.md')	
 	dest.write(c)!
+	redis.hset('doctree:${col_name}', "errors", "errors.md")!
 }

lib/data/doctree/collection/export.v

@@ -1,6 +1,7 @@
 module collection
 
 import freeflowuniverse.herolib.core.pathlib
+import freeflowuniverse.herolib.core.base
 import freeflowuniverse.herolib.core.texttools.regext
 import os
 import freeflowuniverse.herolib.data.doctree.pointer
@@ -15,6 +16,7 @@ pub mut:
 	keep_structure bool // wether the structure of the src collection will be preserved or not
 	exclude_errors bool // wether error reporting should be exported as well
 	replacer       ?regext.ReplaceInstructions
+	redis          bool = true
 }
 
 pub fn (mut c Collection) export(args CollectionExportArgs) ! {
@@ -23,19 +25,24 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! {
 	mut cfile := pathlib.get_file(path: dir_src.path + '/.collection', create: true)! // will auto save it
 	cfile.write("name:${c.name} src:'${c.path.path}'")!
 
-	c.errors << export_pages(c.path.path, c.pages.values(),
+	mut context := base.context()!
+	mut redis := context.redis()!
+	redis.hset('collections:path', '${c.name}', dir_src.path)!
+
+	c.errors << export_pages(c.name, c.path.path, c.pages.values(),
 		dir_src:        dir_src
 		file_paths:     args.file_paths
 		keep_structure: args.keep_structure
 		replacer:       args.replacer
+		redis: args.redis
 	)!
 
-	c.export_files(dir_src, args.reset)!
+	c.export_files(c.name,dir_src, args.reset)!
-	c.export_images(dir_src, args.reset)!
+	c.export_images(c.name,dir_src, args.reset)!
-	c.export_linked_pages(dir_src)!
+	c.export_linked_pages(c.name,dir_src)!
 
 	if !args.exclude_errors {
-		c.errors_report('${dir_src.path}/errors.md')!
+		c.errors_report(c.name,'${dir_src.path}/errors.md')!
 	}
 }
 
@@ -46,11 +53,16 @@ pub mut:
 	file_paths     map[string]string
 	keep_structure bool // wether the structure of the src collection will be preserved or not
 	replacer       ?regext.ReplaceInstructions
+	redis          bool = true
 }
 
 // creates page file, processes page links, then writes page
-fn export_pages(col_path string, pages []&data.Page, args ExportPagesArgs) ![]CollectionError {
+fn export_pages(col_name string, col_path string, pages []&data.Page, args ExportPagesArgs) ![]CollectionError {
 	mut errors := []CollectionError{}
+
+	mut context := base.context()!
+	mut redis := context.redis()!
+
 	for page in pages {
 		dest := if args.keep_structure {
 			relpath := page.path.path.trim_string_left(col_path)
@@ -86,33 +98,43 @@ fn export_pages(col_path string, pages []&data.Page, args ExportPagesArgs) ![]Co
 		if mut replacer := args.replacer {
 			markdown = replacer.replace(text: markdown)!
 		}
-
 		dest_path.write(markdown)!
+		redis.hset('doctree:${col_name}', page.name, "${page.name}.md")!
+
 	}
 	return errors
 }
 
-fn (c Collection) export_files(dir_src pathlib.Path, reset bool) ! {
+fn (c Collection) export_files(col_name string,dir_src pathlib.Path, reset bool) ! {
+	mut context := base.context()!
+	mut redis := context.redis()!	
 	for _, file in c.files {
 		mut d := '${dir_src.path}/img/${file.name}.${file.ext}'
 		if reset || !os.exists(d) {
 			file.copy(d)!
 		}
+		redis.hset('doctree:${col_name}', file.name, "img/${file.name}.${file.ext}")!
 	}
 }
 
-fn (c Collection) export_images(dir_src pathlib.Path, reset bool) ! {
+fn (c Collection) export_images(col_name string,dir_src pathlib.Path, reset bool) ! {
+	mut context := base.context()!
+	mut redis := context.redis()!	
 	for _, file in c.images {
 		mut d := '${dir_src.path}/img/${file.name}.${file.ext}'
+		redis.hset('doctree:${col_name}', file.name, "img/${file.name}.${file.ext}")!
 		if reset || !os.exists(d) {
 			file.copy(d)!
 		}
 	}
 }
 
-fn (c Collection) export_linked_pages(dir_src pathlib.Path) ! {
+fn (c Collection) export_linked_pages(col_name string,dir_src pathlib.Path) ! {
+	mut context := base.context()!
+	mut redis := context.redis()!	
 	collection_linked_pages := c.get_collection_linked_pages()!
 	mut linked_pages_file := pathlib.get_file(path: dir_src.path + '/.linkedpages', create: true)!
+	redis.hset('doctree:${col_name}', "linkedpages", "${linked_pages_file.name()}.md")!
 	linked_pages_file.write(collection_linked_pages.join_lines())!
 }
 

lib/data/doctree/export.v

@@ -14,6 +14,7 @@ pub mut:
 	exclude_errors bool // wether error reporting should be exported as well
 	toreplace      string
 	concurrent     bool = true
+	redis		  bool = true
 }
 
 // export all collections to chosen directory .
@@ -50,6 +51,7 @@ pub fn (mut tree Tree) export(args TreeExportArgs) ! {
 					reset:          args.reset
 					keep_structure: args.keep_structure
 					exclude_errors: args.exclude_errors
+					redis: args.redis
 					// TODO: replacer: tree.replacer
 				)!
 			}(mut col, dest_path, file_paths, args)
@@ -66,6 +68,7 @@ pub fn (mut tree Tree) export(args TreeExportArgs) ! {
 				keep_structure: args.keep_structure
 				exclude_errors: args.exclude_errors
 				replacer:       tree.replacer
+				redis: args.redis
 			)!
 		}
 	}

lib/develop/gittools/repository_load.v

@@ -56,7 +56,6 @@ fn (mut repo GitRepo) load() ! {
 
 // Helper to load remote tags
 fn (mut repo GitRepo) load_branches() ! {
-	println("SDSDSd")
 	tags_result := repo.exec("git for-each-ref --format='%(objectname) %(refname:short)' refs/heads refs/remotes/origin") or {
 		return error('Failed to get branch references: ${err}. Command: git for-each-ref')
 	}

lib/web/doctreeclient/client.v

@@ -0,0 +1,181 @@
+module doctreeclient
+
+import freeflowuniverse.herolib.core.pathlib
+import os
+
+// Error types for DocTreeClient
+pub enum DocTreeError {
+	collection_not_found
+	page_not_found
+	file_not_found
+	image_not_found
+}
+
+// get_page_path returns the path for a page in a collection
+pub fn (c &DocTreeClient) get_page_path(collection_name string, page_name string) !string {
+	// Check if the collection exists
+	collection_path := c.redis.hget('doctree:meta', collection_name) or {
+		return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found')
+	}
+	
+	// Get the relative path of the page within the collection
+	rel_path := c.redis.hget('doctree:${collection_name}', page_name) or {
+		return error('${DocTreeError.page_not_found}: Page "${page_name}" not found in collection "${collection_name}"')
+	}
+	
+	// Combine the collection path with the relative path
+	return os.join_path(collection_path, rel_path)
+}
+
+// get_file_path returns the path for a file in a collection
+pub fn (c &DocTreeClient) get_file_path(collection_name string, file_name string) !string {
+	// Check if the collection exists
+	collection_path := c.redis.hget('doctree:meta', collection_name) or {
+		return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found')
+	}
+	
+	// Get the relative path of the file within the collection
+	rel_path := c.redis.hget('doctree:${collection_name}', file_name) or {
+		return error('${DocTreeError.file_not_found}: File "${file_name}" not found in collection "${collection_name}"')
+	}
+	
+	// Combine the collection path with the relative path
+	return os.join_path(collection_path, rel_path)
+}
+
+// get_image_path returns the path for an image in a collection
+pub fn (c &DocTreeClient) get_image_path(collection_name string, image_name string) !string {
+	// Check if the collection exists
+	collection_path := c.redis.hget('doctree:meta', collection_name) or {
+		return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found')
+	}
+	
+	// Get the relative path of the image within the collection
+	rel_path := c.redis.hget('doctree:${collection_name}', image_name) or {
+		return error('${DocTreeError.image_not_found}: Image "${image_name}" not found in collection "${collection_name}"')
+	}
+	
+	// Combine the collection path with the relative path
+	return os.join_path(collection_path, rel_path)
+}
+
+// page_exists checks if a page exists in a collection
+pub fn (c &DocTreeClient) page_exists(collection_name string, page_name string) bool {
+	// Check if the collection exists
+	if !c.redis.hexists('doctree:meta', collection_name) {
+		return false
+	}
+	
+	// Check if the page exists in the collection
+	return c.redis.hexists('doctree:${collection_name}', page_name)
+}
+
+// file_exists checks if a file exists in a collection
+pub fn (c &DocTreeClient) file_exists(collection_name string, file_name string) bool {
+	// Check if the collection exists
+	if !c.redis.hexists('doctree:meta', collection_name) {
+		return false
+	}
+	
+	// Check if the file exists in the collection
+	return c.redis.hexists('doctree:${collection_name}', file_name)
+}
+
+// image_exists checks if an image exists in a collection
+pub fn (c &DocTreeClient) image_exists(collection_name string, image_name string) bool {
+	// Check if the collection exists
+	if !c.redis.hexists('doctree:meta', collection_name) {
+		return false
+	}
+	
+	// Check if the image exists in the collection
+	return c.redis.hexists('doctree:${collection_name}', image_name)
+}
+
+// get_page_content returns the content of a page in a collection
+pub fn (c &DocTreeClient) get_page_content(collection_name string, page_name string) !string {
+	// Get the path for the page
+	page_path := c.get_page_path(collection_name, page_name)!
+	
+	// Use pathlib to read the file content
+	mut path := pathlib.get_file(path: page_path)!
+	
+	// Check if the file exists
+	if !path.exists() {
+		return error('${DocTreeError.page_not_found}: Page file "${page_path}" does not exist on disk')
+	}
+	
+	// Read and return the file content
+	return path.read()!
+}
+
+// list_collections returns a list of all collection names
+pub fn (c &DocTreeClient) list_collections() ![]string {
+	// Get all collection names from Redis
+	return c.redis.hkeys('doctree:meta')!
+}
+
+// list_pages returns a list of all page names in a collection
+pub fn (c &DocTreeClient) list_pages(collection_name string) ![]string {
+	// Check if the collection exists
+	if !c.redis.hexists('doctree:meta', collection_name) {
+		return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found')
+	}
+	
+	// Get all keys from the collection hash
+	all_keys := c.redis.hkeys('doctree:${collection_name}')!
+	
+	// Filter out only the page names (those without file extensions)
+	mut page_names := []string{}
+	for key in all_keys {
+		if !key.contains('.') {
+			page_names << key
+		}
+	}
+	
+	return page_names
+}
+
+// list_files returns a list of all file names in a collection
+pub fn (c &DocTreeClient) list_files(collection_name string) ![]string {
+	// Check if the collection exists
+	if !c.redis.hexists('doctree:meta', collection_name) {
+		return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found')
+	}
+	
+	// Get all keys from the collection hash
+	all_keys := c.redis.hkeys('doctree:${collection_name}')!
+	
+	// Filter out only the file names (those with file extensions, but not images)
+	mut file_names := []string{}
+	for key in all_keys {
+		if key.contains('.') && !key.ends_with('.png') && !key.ends_with('.jpg') && 
+		   !key.ends_with('.jpeg') && !key.ends_with('.gif') && !key.ends_with('.svg') {
+			file_names << key
+		}
+	}
+	
+	return file_names
+}
+
+// list_images returns a list of all image names in a collection
+pub fn (c &DocTreeClient) list_images(collection_name string) ![]string {
+	// Check if the collection exists
+	if !c.redis.hexists('doctree:meta', collection_name) {
+		return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found')
+	}
+	
+	// Get all keys from the collection hash
+	all_keys := c.redis.hkeys('doctree:${collection_name}')!
+	
+	// Filter out only the image names (those with image extensions)
+	mut image_names := []string{}
+	for key in all_keys {
+		if key.ends_with('.png') || key.ends_with('.jpg') || key.ends_with('.jpeg') || 
+		   key.ends_with('.gif') || key.ends_with('.svg') {
+			image_names << key
+		}
+	}
+	
+	return image_names
+}

lib/web/doctreeclient/doctree_test.v

@@ -0,0 +1,143 @@
+module doctreeclient
+
+import freeflowuniverse.herolib.data.doctree
+import freeflowuniverse.herolib.core.base
+import freeflowuniverse.herolib.core.pathlib
+import os
+
+fn test_doctree_client() ! {
+	println('Setting up doctree data in Redis...')
+	
+	// First, populate Redis with doctree data
+	mut tree := doctree.new(name: 'test')!
+
+	tree.scan(
+		git_url:  'https://git.ourworld.tf/tfgrid/docs_tfgrid4/src/branch/main/collections'
+		git_pull: false
+	)!
+
+	tree.export(
+		destination: '/tmp/mdexport'
+		reset:       true
+		exclude_errors: false
+	)!
+	
+	println('Doctree data populated in Redis')
+	
+	// Create a DocTreeClient instance
+	mut client := new('/tmp/mdexport')!
+	
+	// Test listing collections
+	println('\nListing collections:')
+	collections := client.list_collections()!
+	println('Found ${collections.len} collections')
+	
+	if collections.len == 0 {
+		println('No collections found. Test cannot continue.')
+		return
+	}
+	
+	// Use the first collection for testing
+	collection_name := collections[0]
+	println('\nUsing collection: ${collection_name}')
+	
+	// Test listing pages
+	println('\nListing pages:')
+	pages := client.list_pages(collection_name)!
+	println('Found ${pages.len} pages')
+	
+	if pages.len > 0 {
+		// Test getting page path and content
+		page_name := pages[0]
+		println('\nTesting page: ${page_name}')
+		
+		// Test page existence
+		exists := client.page_exists(collection_name, page_name)
+		println('Page exists: ${exists}')
+		
+		// Test getting page path
+		page_path := client.get_page_path(collection_name, page_name)!
+		println('Page path: ${page_path}')
+		
+		// Test getting page content
+		content := client.get_page_content(collection_name, page_name)!
+		println('Page content length: ${content.len} characters')
+		println('First 100 characters: ${content[..min(100, content.len)]}...')
+	} else {
+		println('No pages found for testing')
+	}
+	
+	// Test listing images
+	println('\nListing images:')
+	images := client.list_images(collection_name)!
+	println('Found ${images.len} images')
+	
+	if images.len > 0 {
+		// Test getting image path
+		image_name := images[0]
+		println('\nTesting image: ${image_name}')
+		
+		// Test image existence
+		exists := client.image_exists(collection_name, image_name)
+		println('Image exists: ${exists}')
+		
+		// Test getting image path
+		image_path := client.get_image_path(collection_name, image_name)!
+		println('Image path: ${image_path}')
+		
+		// Check if the image file exists on disk
+		println('Image file exists on disk: ${os.exists(image_path)}')
+	} else {
+		println('No images found for testing')
+	}
+	
+	// Test listing files
+	println('\nListing files:')
+	files := client.list_files(collection_name)!
+	println('Found ${files.len} files')
+	
+	if files.len > 0 {
+		// Test getting file path
+		file_name := files[0]
+		println('\nTesting file: ${file_name}')
+		
+		// Test file existence
+		exists := client.file_exists(collection_name, file_name)
+		println('File exists: ${exists}')
+		
+		// Test getting file path
+		file_path := client.get_file_path(collection_name, file_name)!
+		println('File path: ${file_path}')
+		
+		// Check if the file exists on disk
+		println('File exists on disk: ${os.exists(file_path)}')
+	} else {
+		println('No files found for testing')
+	}
+	
+	// Test error handling
+	println('\nTesting error handling:')
+	
+	// Test with non-existent collection
+	non_existent_collection := 'non_existent_collection'
+	println('Testing with non-existent collection: ${non_existent_collection}')
+	
+	exists := client.page_exists(non_existent_collection, 'any_page')
+	println('Page exists in non-existent collection: ${exists} (should be false)')
+	
+	// Test with non-existent page
+	non_existent_page := 'non_existent_page'
+	println('Testing with non-existent page: ${non_existent_page}')
+	
+	exists2 := client.page_exists(collection_name, non_existent_page)
+	println('Non-existent page exists: ${exists2} (should be false)')
+	
+	println('\nTest completed successfully!')
+}
+
+fn main() {
+	test_doctree_client() or {
+		eprintln('Error: ${err}')
+		exit(1)
+	}
+}

lib/web/doctreeclient/factory.v

@@ -0,0 +1,14 @@
+module doctreeclient
+import freeflowuniverse.herolib.core.base
+
+// new creates a new DocTreeClient instance
+// path: The base path where doctree collections are exported (not used internally but kept for API consistency)
+pub fn new(path string) !&DocTreeClient {
+	mut context := base.context()!
+	mut redis := context.redis()!
+
+	return &DocTreeClient{
+		redis: redis
+	}
+}
+

lib/web/doctreeclient/model.v

@@ -0,0 +1,9 @@
+module doctreeclient
+
+import freeflowuniverse.herolib.core.redisclient
+
+// Combined config structure
+pub struct DocTreeClient {
+pub mut:
+	redis		  &redisclient.Redis
+}

lib/web/siteconfig/config.v

@@ -22,9 +22,10 @@ pub mut:
 	title      string
 	description string
 	draft	   bool
-	folder 	  string
+	folder 	   string
 	prio	   int
-	src   string
+	src   	   string
+	collection string
 }
 
 // Footer config structures