From b410544ee17638a5813d9746799f22b1cf2c3fa2 Mon Sep 17 00:00:00 2001 From: despiegk Date: Wed, 21 May 2025 08:30:30 +0400 Subject: [PATCH] ... --- examples/data/encrypt_decrypt.vsh | 2 +- .../mdbook_markdown/doctree_export.vsh | 11 +- lib/data/doctree/README.md | 86 +++++++++ lib/data/doctree/collection/error.v | 8 +- lib/data/doctree/collection/export.v | 42 +++- lib/data/doctree/export.v | 3 + lib/develop/gittools/repository_load.v | 1 - lib/web/doctreeclient/client.v | 181 ++++++++++++++++++ lib/web/doctreeclient/doctree_test.v | 143 ++++++++++++++ lib/web/doctreeclient/factory.v | 14 ++ lib/web/doctreeclient/model.v | 9 + lib/web/siteconfig/config.v | 5 +- 12 files changed, 482 insertions(+), 23 deletions(-) create mode 100644 lib/data/doctree/README.md create mode 100644 lib/web/doctreeclient/client.v create mode 100644 lib/web/doctreeclient/doctree_test.v create mode 100644 lib/web/doctreeclient/factory.v create mode 100644 lib/web/doctreeclient/model.v diff --git a/examples/data/encrypt_decrypt.vsh b/examples/data/encrypt_decrypt.vsh index 74a38407..1537ca72 100755 --- a/examples/data/encrypt_decrypt.vsh +++ b/examples/data/encrypt_decrypt.vsh @@ -1,4 +1,4 @@ -#!/usr/bin/env -S v -n -w -gc none -cc tcc -d use_openssl -enable-globals run +#!/usr/bin/env -S v -n -w -gc none -cg -cc tcc -d use_openssl -enable-globals run import freeflowuniverse.herolib.crypt.aes_symmetric { decrypt, encrypt } import freeflowuniverse.herolib.ui.console diff --git a/examples/webtools/mdbook_markdown/doctree_export.vsh b/examples/webtools/mdbook_markdown/doctree_export.vsh index 9b2eca0e..068363b6 100755 --- a/examples/webtools/mdbook_markdown/doctree_export.vsh +++ b/examples/webtools/mdbook_markdown/doctree_export.vsh @@ -10,13 +10,10 @@ mut tree := doctree.new(name: 'test')! // git_reset bool // git_root string // git_pull bool -// load bool = true // means we scan automatically the added collection -for project in 'projectinca, legal, why'.split(',').map(it.trim_space()) { - tree.scan( - git_url: 'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/development/collections/${project}' - git_pull: false - )! -} +tree.scan( + git_url: 'https://git.ourworld.tf/tfgrid/docs_tfgrid4/src/branch/main/collections' + git_pull: false +)! tree.export( destination: '/tmp/mdexport' diff --git a/lib/data/doctree/README.md b/lib/data/doctree/README.md new file mode 100644 index 00000000..0ee82435 --- /dev/null +++ b/lib/data/doctree/README.md @@ -0,0 +1,86 @@ +# Doctree Module + +The `doctree` module is a V language library designed for scanning, processing, and exporting collections of documents. It provides a structured way to manage document-based content, making it suitable for generating documentation, building static websites, or processing any content organized into collections. + +## Purpose + +The primary goal of this module is to transform structured document collections into a format suitable for various outputs. It handles the complexities of finding collections, loading their content, processing includes, definitions, and macros, and exporting the final result while managing assets like images and files. + +## Key Concepts + +* **Tree:** The central component (`doctree.Tree`) that holds one or more `Collection` instances. It orchestrates the scanning, processing, and exporting of all contained collections. +* **Collection:** A directory that is marked as a collection by the presence of a `.collection` file. A collection groups related documents (pages, images, files) and can have its own configuration defined within the `.collection` file. +* **.collection file:** A file placed in a directory to designate it as a collection. This file can optionally contain parameters (using the `paramsparser` format) such as a custom name for the collection. + +## How it Works (Workflow) + +The typical workflow involves creating a `Tree`, scanning for collections, and then exporting the processed content.å + +1. **Create Tree:** Initialize a `doctree.Tree` instance using `doctree.new()`. +2. **Scan:** Use the `tree.scan()` or `tree.scan_concurrent()` method, providing a path to a directory or a Git repository URL. The scanner recursively looks for directories containing a `.collection` file. +3. **Load Content:** For each identified collection, the module loads its content, including markdown pages, images, and other files. +4. **Process Content:** The loaded content is processed. This includes handling definitions, includes (content from other files), and macros (dynamic content generation or transformation). +5. **Generate Output Paths:** The module determines the final paths for all processed files and assets in the destination directory. +6. **Export:** The `tree.export()` method writes the processed content and assets to the specified destination directory, maintaining the desired structure. + +## Usage (For Developers) + +Here's a basic example of how to use the `doctree` module in your V project: + +```v +import freeflowuniverse.herolib.data.doctree +// 1. Create a new Tree instance +mut tree := doctree.new(name: 'my_documentation')! + +// 2. Scan a directory containing your collections +// Replace './docs' with the actual path to your document collections +tree.scan(path: './docs')! + +// use from URL +//git_url string +//git_reset bool +//git_pull bool +tree.scan(git_url: 'https://git.ourworld.tf/tfgrid/docs_tfgrid4/src/branch/main/collections')! + +// 3. Export the processed content to a destination directory +// Replace './output' with your desired output path +// if redis then the metadata will be put in redis +tree.export(destination: './output', redis:true)! + +println('Documentation successfully exported to ./output') + +``` + +## Structure of a Collection + +A collection is a directory containing a `.collection` file. Inside a collection directory, you would typically organize your content like this: + +``` +my_collection/ +├── .collection +├── page1.md +├── page2.md +├── images/ +│ ├── image1.png +│ └── image2.jpg +└── files/ + ├── document.pdf + └── data.csv +``` + +Markdown files (`.md`) are treated as pages. + +## Redis Structure + +when using the export redis:true argument, which is default + +in redis we will find + +```bash +#redis hsets: +doctree:$collectionname $pagename $rel_path_in_collection +doctree:$collectionname $filename.$ext $rel_path_in_collection +doctree:meta $collectionname $collectionpath_on_disk +``` + + diff --git a/lib/data/doctree/collection/error.v b/lib/data/doctree/collection/error.v index 53f9b18b..e8bfc1c2 100644 --- a/lib/data/doctree/collection/error.v +++ b/lib/data/doctree/collection/error.v @@ -1,6 +1,7 @@ module collection import freeflowuniverse.herolib.core.pathlib { Path } +import freeflowuniverse.herolib.core.base import freeflowuniverse.herolib.ui.console pub enum CollectionErrorCat { @@ -52,13 +53,16 @@ pub fn (err ObjNotFound) msg() string { } // write errors.md in the collection, this allows us to see what the errors are -pub fn (collection Collection) errors_report(dest_ string) ! { +pub fn (collection Collection) errors_report(col_name string,dest_ string) ! { // console.print_debug("====== errors report: ${dest_} : ${collection.errors.len}\n${collection.errors}") + mut context := base.context()! + mut redis := context.redis()! mut dest := pathlib.get_file(path: dest_, create: true)! if collection.errors.len == 0 { dest.delete()! return } - c := $tmpl('template/errors.md') + c := $tmpl('template/errors.md') dest.write(c)! + redis.hset('doctree:${col_name}', "errors", "errors.md")! } diff --git a/lib/data/doctree/collection/export.v b/lib/data/doctree/collection/export.v index a28fc3c6..8636480c 100644 --- a/lib/data/doctree/collection/export.v +++ b/lib/data/doctree/collection/export.v @@ -1,6 +1,7 @@ module collection import freeflowuniverse.herolib.core.pathlib +import freeflowuniverse.herolib.core.base import freeflowuniverse.herolib.core.texttools.regext import os import freeflowuniverse.herolib.data.doctree.pointer @@ -15,6 +16,7 @@ pub mut: keep_structure bool // wether the structure of the src collection will be preserved or not exclude_errors bool // wether error reporting should be exported as well replacer ?regext.ReplaceInstructions + redis bool = true } pub fn (mut c Collection) export(args CollectionExportArgs) ! { @@ -23,19 +25,24 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! { mut cfile := pathlib.get_file(path: dir_src.path + '/.collection', create: true)! // will auto save it cfile.write("name:${c.name} src:'${c.path.path}'")! - c.errors << export_pages(c.path.path, c.pages.values(), + mut context := base.context()! + mut redis := context.redis()! + redis.hset('collections:path', '${c.name}', dir_src.path)! + + c.errors << export_pages(c.name, c.path.path, c.pages.values(), dir_src: dir_src file_paths: args.file_paths keep_structure: args.keep_structure replacer: args.replacer + redis: args.redis )! - c.export_files(dir_src, args.reset)! - c.export_images(dir_src, args.reset)! - c.export_linked_pages(dir_src)! + c.export_files(c.name,dir_src, args.reset)! + c.export_images(c.name,dir_src, args.reset)! + c.export_linked_pages(c.name,dir_src)! if !args.exclude_errors { - c.errors_report('${dir_src.path}/errors.md')! + c.errors_report(c.name,'${dir_src.path}/errors.md')! } } @@ -46,11 +53,16 @@ pub mut: file_paths map[string]string keep_structure bool // wether the structure of the src collection will be preserved or not replacer ?regext.ReplaceInstructions + redis bool = true } // creates page file, processes page links, then writes page -fn export_pages(col_path string, pages []&data.Page, args ExportPagesArgs) ![]CollectionError { +fn export_pages(col_name string, col_path string, pages []&data.Page, args ExportPagesArgs) ![]CollectionError { mut errors := []CollectionError{} + + mut context := base.context()! + mut redis := context.redis()! + for page in pages { dest := if args.keep_structure { relpath := page.path.path.trim_string_left(col_path) @@ -86,33 +98,43 @@ fn export_pages(col_path string, pages []&data.Page, args ExportPagesArgs) ![]Co if mut replacer := args.replacer { markdown = replacer.replace(text: markdown)! } - dest_path.write(markdown)! + redis.hset('doctree:${col_name}', page.name, "${page.name}.md")! + } return errors } -fn (c Collection) export_files(dir_src pathlib.Path, reset bool) ! { +fn (c Collection) export_files(col_name string,dir_src pathlib.Path, reset bool) ! { + mut context := base.context()! + mut redis := context.redis()! for _, file in c.files { mut d := '${dir_src.path}/img/${file.name}.${file.ext}' if reset || !os.exists(d) { file.copy(d)! } + redis.hset('doctree:${col_name}', file.name, "img/${file.name}.${file.ext}")! } } -fn (c Collection) export_images(dir_src pathlib.Path, reset bool) ! { +fn (c Collection) export_images(col_name string,dir_src pathlib.Path, reset bool) ! { + mut context := base.context()! + mut redis := context.redis()! for _, file in c.images { mut d := '${dir_src.path}/img/${file.name}.${file.ext}' + redis.hset('doctree:${col_name}', file.name, "img/${file.name}.${file.ext}")! if reset || !os.exists(d) { file.copy(d)! } } } -fn (c Collection) export_linked_pages(dir_src pathlib.Path) ! { +fn (c Collection) export_linked_pages(col_name string,dir_src pathlib.Path) ! { + mut context := base.context()! + mut redis := context.redis()! collection_linked_pages := c.get_collection_linked_pages()! mut linked_pages_file := pathlib.get_file(path: dir_src.path + '/.linkedpages', create: true)! + redis.hset('doctree:${col_name}', "linkedpages", "${linked_pages_file.name()}.md")! linked_pages_file.write(collection_linked_pages.join_lines())! } diff --git a/lib/data/doctree/export.v b/lib/data/doctree/export.v index ad228407..1ab7f4e5 100644 --- a/lib/data/doctree/export.v +++ b/lib/data/doctree/export.v @@ -14,6 +14,7 @@ pub mut: exclude_errors bool // wether error reporting should be exported as well toreplace string concurrent bool = true + redis bool = true } // export all collections to chosen directory . @@ -50,6 +51,7 @@ pub fn (mut tree Tree) export(args TreeExportArgs) ! { reset: args.reset keep_structure: args.keep_structure exclude_errors: args.exclude_errors + redis: args.redis // TODO: replacer: tree.replacer )! }(mut col, dest_path, file_paths, args) @@ -66,6 +68,7 @@ pub fn (mut tree Tree) export(args TreeExportArgs) ! { keep_structure: args.keep_structure exclude_errors: args.exclude_errors replacer: tree.replacer + redis: args.redis )! } } diff --git a/lib/develop/gittools/repository_load.v b/lib/develop/gittools/repository_load.v index da3f0f60..ce692ff0 100644 --- a/lib/develop/gittools/repository_load.v +++ b/lib/develop/gittools/repository_load.v @@ -56,7 +56,6 @@ fn (mut repo GitRepo) load() ! { // Helper to load remote tags fn (mut repo GitRepo) load_branches() ! { - println("SDSDSd") tags_result := repo.exec("git for-each-ref --format='%(objectname) %(refname:short)' refs/heads refs/remotes/origin") or { return error('Failed to get branch references: ${err}. Command: git for-each-ref') } diff --git a/lib/web/doctreeclient/client.v b/lib/web/doctreeclient/client.v new file mode 100644 index 00000000..b825bedc --- /dev/null +++ b/lib/web/doctreeclient/client.v @@ -0,0 +1,181 @@ +module doctreeclient + +import freeflowuniverse.herolib.core.pathlib +import os + +// Error types for DocTreeClient +pub enum DocTreeError { + collection_not_found + page_not_found + file_not_found + image_not_found +} + +// get_page_path returns the path for a page in a collection +pub fn (c &DocTreeClient) get_page_path(collection_name string, page_name string) !string { + // Check if the collection exists + collection_path := c.redis.hget('doctree:meta', collection_name) or { + return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found') + } + + // Get the relative path of the page within the collection + rel_path := c.redis.hget('doctree:${collection_name}', page_name) or { + return error('${DocTreeError.page_not_found}: Page "${page_name}" not found in collection "${collection_name}"') + } + + // Combine the collection path with the relative path + return os.join_path(collection_path, rel_path) +} + +// get_file_path returns the path for a file in a collection +pub fn (c &DocTreeClient) get_file_path(collection_name string, file_name string) !string { + // Check if the collection exists + collection_path := c.redis.hget('doctree:meta', collection_name) or { + return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found') + } + + // Get the relative path of the file within the collection + rel_path := c.redis.hget('doctree:${collection_name}', file_name) or { + return error('${DocTreeError.file_not_found}: File "${file_name}" not found in collection "${collection_name}"') + } + + // Combine the collection path with the relative path + return os.join_path(collection_path, rel_path) +} + +// get_image_path returns the path for an image in a collection +pub fn (c &DocTreeClient) get_image_path(collection_name string, image_name string) !string { + // Check if the collection exists + collection_path := c.redis.hget('doctree:meta', collection_name) or { + return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found') + } + + // Get the relative path of the image within the collection + rel_path := c.redis.hget('doctree:${collection_name}', image_name) or { + return error('${DocTreeError.image_not_found}: Image "${image_name}" not found in collection "${collection_name}"') + } + + // Combine the collection path with the relative path + return os.join_path(collection_path, rel_path) +} + +// page_exists checks if a page exists in a collection +pub fn (c &DocTreeClient) page_exists(collection_name string, page_name string) bool { + // Check if the collection exists + if !c.redis.hexists('doctree:meta', collection_name) { + return false + } + + // Check if the page exists in the collection + return c.redis.hexists('doctree:${collection_name}', page_name) +} + +// file_exists checks if a file exists in a collection +pub fn (c &DocTreeClient) file_exists(collection_name string, file_name string) bool { + // Check if the collection exists + if !c.redis.hexists('doctree:meta', collection_name) { + return false + } + + // Check if the file exists in the collection + return c.redis.hexists('doctree:${collection_name}', file_name) +} + +// image_exists checks if an image exists in a collection +pub fn (c &DocTreeClient) image_exists(collection_name string, image_name string) bool { + // Check if the collection exists + if !c.redis.hexists('doctree:meta', collection_name) { + return false + } + + // Check if the image exists in the collection + return c.redis.hexists('doctree:${collection_name}', image_name) +} + +// get_page_content returns the content of a page in a collection +pub fn (c &DocTreeClient) get_page_content(collection_name string, page_name string) !string { + // Get the path for the page + page_path := c.get_page_path(collection_name, page_name)! + + // Use pathlib to read the file content + mut path := pathlib.get_file(path: page_path)! + + // Check if the file exists + if !path.exists() { + return error('${DocTreeError.page_not_found}: Page file "${page_path}" does not exist on disk') + } + + // Read and return the file content + return path.read()! +} + +// list_collections returns a list of all collection names +pub fn (c &DocTreeClient) list_collections() ![]string { + // Get all collection names from Redis + return c.redis.hkeys('doctree:meta')! +} + +// list_pages returns a list of all page names in a collection +pub fn (c &DocTreeClient) list_pages(collection_name string) ![]string { + // Check if the collection exists + if !c.redis.hexists('doctree:meta', collection_name) { + return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found') + } + + // Get all keys from the collection hash + all_keys := c.redis.hkeys('doctree:${collection_name}')! + + // Filter out only the page names (those without file extensions) + mut page_names := []string{} + for key in all_keys { + if !key.contains('.') { + page_names << key + } + } + + return page_names +} + +// list_files returns a list of all file names in a collection +pub fn (c &DocTreeClient) list_files(collection_name string) ![]string { + // Check if the collection exists + if !c.redis.hexists('doctree:meta', collection_name) { + return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found') + } + + // Get all keys from the collection hash + all_keys := c.redis.hkeys('doctree:${collection_name}')! + + // Filter out only the file names (those with file extensions, but not images) + mut file_names := []string{} + for key in all_keys { + if key.contains('.') && !key.ends_with('.png') && !key.ends_with('.jpg') && + !key.ends_with('.jpeg') && !key.ends_with('.gif') && !key.ends_with('.svg') { + file_names << key + } + } + + return file_names +} + +// list_images returns a list of all image names in a collection +pub fn (c &DocTreeClient) list_images(collection_name string) ![]string { + // Check if the collection exists + if !c.redis.hexists('doctree:meta', collection_name) { + return error('${DocTreeError.collection_not_found}: Collection "${collection_name}" not found') + } + + // Get all keys from the collection hash + all_keys := c.redis.hkeys('doctree:${collection_name}')! + + // Filter out only the image names (those with image extensions) + mut image_names := []string{} + for key in all_keys { + if key.ends_with('.png') || key.ends_with('.jpg') || key.ends_with('.jpeg') || + key.ends_with('.gif') || key.ends_with('.svg') { + image_names << key + } + } + + return image_names +} diff --git a/lib/web/doctreeclient/doctree_test.v b/lib/web/doctreeclient/doctree_test.v new file mode 100644 index 00000000..c744f47e --- /dev/null +++ b/lib/web/doctreeclient/doctree_test.v @@ -0,0 +1,143 @@ +module doctreeclient + +import freeflowuniverse.herolib.data.doctree +import freeflowuniverse.herolib.core.base +import freeflowuniverse.herolib.core.pathlib +import os + +fn test_doctree_client() ! { + println('Setting up doctree data in Redis...') + + // First, populate Redis with doctree data + mut tree := doctree.new(name: 'test')! + + tree.scan( + git_url: 'https://git.ourworld.tf/tfgrid/docs_tfgrid4/src/branch/main/collections' + git_pull: false + )! + + tree.export( + destination: '/tmp/mdexport' + reset: true + exclude_errors: false + )! + + println('Doctree data populated in Redis') + + // Create a DocTreeClient instance + mut client := new('/tmp/mdexport')! + + // Test listing collections + println('\nListing collections:') + collections := client.list_collections()! + println('Found ${collections.len} collections') + + if collections.len == 0 { + println('No collections found. Test cannot continue.') + return + } + + // Use the first collection for testing + collection_name := collections[0] + println('\nUsing collection: ${collection_name}') + + // Test listing pages + println('\nListing pages:') + pages := client.list_pages(collection_name)! + println('Found ${pages.len} pages') + + if pages.len > 0 { + // Test getting page path and content + page_name := pages[0] + println('\nTesting page: ${page_name}') + + // Test page existence + exists := client.page_exists(collection_name, page_name) + println('Page exists: ${exists}') + + // Test getting page path + page_path := client.get_page_path(collection_name, page_name)! + println('Page path: ${page_path}') + + // Test getting page content + content := client.get_page_content(collection_name, page_name)! + println('Page content length: ${content.len} characters') + println('First 100 characters: ${content[..min(100, content.len)]}...') + } else { + println('No pages found for testing') + } + + // Test listing images + println('\nListing images:') + images := client.list_images(collection_name)! + println('Found ${images.len} images') + + if images.len > 0 { + // Test getting image path + image_name := images[0] + println('\nTesting image: ${image_name}') + + // Test image existence + exists := client.image_exists(collection_name, image_name) + println('Image exists: ${exists}') + + // Test getting image path + image_path := client.get_image_path(collection_name, image_name)! + println('Image path: ${image_path}') + + // Check if the image file exists on disk + println('Image file exists on disk: ${os.exists(image_path)}') + } else { + println('No images found for testing') + } + + // Test listing files + println('\nListing files:') + files := client.list_files(collection_name)! + println('Found ${files.len} files') + + if files.len > 0 { + // Test getting file path + file_name := files[0] + println('\nTesting file: ${file_name}') + + // Test file existence + exists := client.file_exists(collection_name, file_name) + println('File exists: ${exists}') + + // Test getting file path + file_path := client.get_file_path(collection_name, file_name)! + println('File path: ${file_path}') + + // Check if the file exists on disk + println('File exists on disk: ${os.exists(file_path)}') + } else { + println('No files found for testing') + } + + // Test error handling + println('\nTesting error handling:') + + // Test with non-existent collection + non_existent_collection := 'non_existent_collection' + println('Testing with non-existent collection: ${non_existent_collection}') + + exists := client.page_exists(non_existent_collection, 'any_page') + println('Page exists in non-existent collection: ${exists} (should be false)') + + // Test with non-existent page + non_existent_page := 'non_existent_page' + println('Testing with non-existent page: ${non_existent_page}') + + exists2 := client.page_exists(collection_name, non_existent_page) + println('Non-existent page exists: ${exists2} (should be false)') + + println('\nTest completed successfully!') +} + +fn main() { + test_doctree_client() or { + eprintln('Error: ${err}') + exit(1) + } +} \ No newline at end of file diff --git a/lib/web/doctreeclient/factory.v b/lib/web/doctreeclient/factory.v new file mode 100644 index 00000000..bc6926e7 --- /dev/null +++ b/lib/web/doctreeclient/factory.v @@ -0,0 +1,14 @@ +module doctreeclient +import freeflowuniverse.herolib.core.base + +// new creates a new DocTreeClient instance +// path: The base path where doctree collections are exported (not used internally but kept for API consistency) +pub fn new(path string) !&DocTreeClient { + mut context := base.context()! + mut redis := context.redis()! + + return &DocTreeClient{ + redis: redis + } +} + diff --git a/lib/web/doctreeclient/model.v b/lib/web/doctreeclient/model.v new file mode 100644 index 00000000..3d4c74a7 --- /dev/null +++ b/lib/web/doctreeclient/model.v @@ -0,0 +1,9 @@ +module doctreeclient + +import freeflowuniverse.herolib.core.redisclient + +// Combined config structure +pub struct DocTreeClient { +pub mut: + redis &redisclient.Redis +} diff --git a/lib/web/siteconfig/config.v b/lib/web/siteconfig/config.v index 2270573f..ce1ab497 100644 --- a/lib/web/siteconfig/config.v +++ b/lib/web/siteconfig/config.v @@ -22,9 +22,10 @@ pub mut: title string description string draft bool - folder string + folder string prio int - src string + src string + collection string } // Footer config structures