From a149845fc7a52c2b4572b17cf3eeef1ddda9264f Mon Sep 17 00:00:00 2001 From: Mahmoud-Emad Date: Thu, 6 Nov 2025 15:44:09 +0200 Subject: [PATCH] feat: Enhance docusaurus site generation with atlas client - Add flags for development server and browser opening - Introduce IDocClient interface for unified client access - Implement atlas_client integration for Docusaurus - Refactor link handling and image path resolution - Update Docusaurus config with atlas client options --- lib/core/herocmds/atlas.v | 42 +++++++- lib/data/atlas/collection.v | 7 +- lib/data/atlas/export.v | 40 +++++++- lib/data/atlas/link.v | 124 ++++++++++++----------- lib/web/atlas_client/client.v | 42 +++++--- lib/web/docusaurus/config.v | 20 ++-- lib/web/docusaurus/dsite_generate_docs.v | 112 +++++++++++++++++++- lib/web/docusaurus/play.v | 12 ++- 8 files changed, 303 insertions(+), 96 deletions(-) diff --git a/lib/core/herocmds/atlas.v b/lib/core/herocmds/atlas.v index 6dfd5181..24ded638 100644 --- a/lib/core/herocmds/atlas.v +++ b/lib/core/herocmds/atlas.v @@ -4,6 +4,7 @@ import incubaid.herolib.ui.console import incubaid.herolib.data.atlas import incubaid.herolib.core.playcmds import incubaid.herolib.develop.gittools +import incubaid.herolib.web.docusaurus import os import cli { Command, Flag } @@ -92,6 +93,21 @@ pub fn cmd_atlas(mut cmdroot Command) Command { description: 'Update environment and git pull before operations.' }) + cmd_run.add_flag(Flag{ + flag: .bool + required: false + name: 'dev' + description: 'Run development server after export (requires docusaurus config).' + }) + + cmd_run.add_flag(Flag{ + flag: .bool + required: false + name: 'open' + abbrev: 'o' + description: 'Open browser when running dev server (use with --dev).' + }) + cmdroot.add_command(cmd_run) return cmdroot } @@ -102,6 +118,8 @@ fn cmd_atlas_execute(cmd Command) ! { mut update := cmd.flags.get_bool('update') or { false } mut scan := cmd.flags.get_bool('scan') or { false } mut export := cmd.flags.get_bool('export') or { false } + mut dev := cmd.flags.get_bool('dev') or { false } + mut open_ := cmd.flags.get_bool('open') or { false } // Include and redis default to true unless explicitly disabled mut no_include := cmd.flags.get_bool('no-include') or { false } @@ -132,7 +150,7 @@ fn cmd_atlas_execute(cmd Command) ! { // Run HeroScript if exists playcmds.run( heroscript_path: atlas_path.path - reset: false + reset: reset emptycheck: false )! @@ -181,5 +199,27 @@ fn cmd_atlas_execute(cmd Command) ! { col.print_errors() } } + + // Run dev server if -dev flag is set + if dev { + console.print_header('Starting development server...') + console.print_item('Atlas export directory: ${destination}') + console.print_item('Looking for docusaurus configuration in: ${atlas_path.path}') + + // Run the docusaurus dev server using the exported atlas content + // This will look for a .heroscript file in the atlas_path that configures docusaurus + // with use_atlas:true and atlas_export_dir pointing to the destination + playcmds.run( + heroscript_path: atlas_path.path + reset: reset + )! + + // Get the docusaurus site and run dev server + mut dsite := docusaurus.dsite_get('')! + dsite.dev( + open: open_ + watch_changes: false + )! + } } } diff --git a/lib/data/atlas/collection.v b/lib/data/atlas/collection.v index 9028c9cb..ad7ca039 100644 --- a/lib/data/atlas/collection.v +++ b/lib/data/atlas/collection.v @@ -70,9 +70,10 @@ fn (mut c Collection) add_page(mut path pathlib.Path) ! { // Add an image to the collection fn (mut c Collection) add_file(mut p pathlib.Path) ! { - name := p.name_fix_keepext() + // Use name without extension for the key and name field + name := p.name_fix_no_ext() if name in c.files { - return error('Page ${name} already exists in collection ${c.name}') + return error('File ${name} already exists in collection ${c.name}') } // Use absolute paths for path_relative to work correctly mut col_path := pathlib.get(c.path) @@ -80,7 +81,7 @@ fn (mut c Collection) add_file(mut p pathlib.Path) ! { relativepath := file_abs_path.path_relative(col_path.absolute())! mut file_new := File{ - name: name + name: name // name without extension ext: p.extension_lower() path: relativepath // relative path of file in the collection collection: &c diff --git a/lib/data/atlas/export.v b/lib/data/atlas/export.v index 3fbb36f5..5ca67447 100644 --- a/lib/data/atlas/export.v +++ b/lib/data/atlas/export.v @@ -67,8 +67,9 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! { )! json_file.write(meta)! - // Track cross-collection pages that need to be copied for self-contained export + // Track cross-collection pages and files that need to be copied for self-contained export mut cross_collection_pages := map[string]&Page{} // key: page.name, value: &Page + mut cross_collection_files := map[string]&File{} // key: file.name, value: &File // First pass: export all pages in this collection and collect cross-collection references for _, mut page in c.pages { @@ -82,17 +83,32 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! { mut dest_file := pathlib.get_file(path: '${col_dir.path}/${page.name}.md', create: true)! dest_file.write(content)! - // Collect cross-collection page references for copying + // Collect cross-collection references for copying (pages and files/images) // IMPORTANT: Use cached links from validation (before transformation) to preserve collection info for mut link in page.links { - // Only process valid page links (not files/images) from other collections - if link.status == .found && !link.is_file_link && !link.is_local_in_collection() { + if link.status != .found { + continue + } + + // Collect cross-collection page references + is_local := link.target_collection_name == c.name + if !link.is_file_link && !is_local { mut target_page := link.target_page() or { continue } // Use page name as key to avoid duplicates if target_page.name !in cross_collection_pages { cross_collection_pages[target_page.name] = target_page } } + + // Collect cross-collection file/image references + if link.is_file_link && !is_local { + mut target_file := link.target_file() or { continue } + // Use file name as key to avoid duplicates + file_key := target_file.file_name() + if file_key !in cross_collection_files { + cross_collection_files[file_key] = target_file + } + } } // Redis operations... @@ -103,6 +119,14 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! { } } + // Copy all files/images from this collection to the export directory + for _, mut file in c.files { + mut src_file := file.path()! + mut dest_path := '${col_dir.path}/${file.file_name()}' + mut dest_file := pathlib.get_file(path: dest_path, create: true)! + src_file.copy(dest: dest_file.path)! + } + // Second pass: copy cross-collection referenced pages to make collection self-contained for _, mut ref_page in cross_collection_pages { // Get the referenced page content with includes processed @@ -116,4 +140,12 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! { mut dest_file := pathlib.get_file(path: '${col_dir.path}/${ref_page.name}.md', create: true)! dest_file.write(ref_content)! } + + // Third pass: copy cross-collection referenced files/images to make collection self-contained + for _, mut ref_file in cross_collection_files { + mut src_file := ref_file.path()! + mut dest_path := '${col_dir.path}/${ref_file.file_name()}' + mut dest_file := pathlib.get_file(path: dest_path, create: true)! + src_file.copy(dest: dest_file.path)! + } } diff --git a/lib/data/atlas/link.v b/lib/data/atlas/link.v index e2683fe5..33ad2c6e 100644 --- a/lib/data/atlas/link.v +++ b/lib/data/atlas/link.v @@ -27,20 +27,12 @@ pub enum LinkStatus { error } +// Get the collection:item key for this link fn (mut self Link) key() string { return '${self.target_collection_name}:${self.target_item_name}' } -// is the link in the same collection as the page containing the link -fn (mut self Link) is_local_in_collection() bool { - return self.target_collection_name == self.page.collection.name -} - -// is the link pointing to an external resource e.g. http, git, mailto, ftp -pub fn (mut self Link) is_external() bool { - return self.status == .external -} - +// Get the target page this link points to pub fn (mut self Link) target_page() !&Page { if self.status == .external { return error('External links do not have a target page') @@ -48,6 +40,7 @@ pub fn (mut self Link) target_page() !&Page { return self.page.collection.atlas.page_get(self.key()) } +// Get the target file this link points to pub fn (mut self Link) target_file() !&File { if self.status == .external { return error('External links do not have a target file') @@ -92,21 +85,23 @@ fn (mut p Page) find_links(content string) ![]Link { text := line[open_bracket + 1..close_bracket] target := line[open_paren + 1..close_paren] - mut is_image_link := (image_open != -1) - - mut is_file_link := false - - // if no . in file then it means it's a page link (binaries with . are not supported in other words) - if target.contains('.') && (!target.trim_space().to_lower().ends_with('.md')) { - is_file_link = true - is_image_link = false // means it's a file link, not an image link - } + // Determine link type + // File links have extensions (but not .md), image links start with ![ + is_file_link := target.contains('.') && !target.trim_space().to_lower().ends_with('.md') + is_image_link := image_open != -1 && !is_file_link // Store position - use image_open if it's an image, otherwise open_bracket link_start_pos := if is_image_link { image_open } else { open_bracket } + // For image links, src should include the ! prefix + link_src := if is_image_link { + line[image_open..close_paren + 1] + } else { + line[open_bracket..close_paren + 1] + } + mut link := Link{ - src: line[open_bracket..close_paren + 1] + src: link_src text: text target: target.trim_space() line: line_idx + 1 @@ -133,23 +128,24 @@ fn (mut p Page) find_links(content string) ![]Link { fn (mut p Page) parse_link_target(mut link Link) { mut target := link.target.to_lower().trim_space() - // Skip external links + // Check for external links (http, https, mailto, ftp) if target.starts_with('http://') || target.starts_with('https://') || target.starts_with('mailto:') || target.starts_with('ftp://') { link.status = .external return } - // Skip anchors + // Check for anchor links if target.starts_with('#') { link.status = .anchor return } + // Handle relative paths - extract the last part after / if target.contains('/') { - parts9 := target.split('/') - if parts9.len >= 1 { - target = parts9[1] + parts := target.split('/') + if parts.len > 1 { + target = parts[parts.len - 1] } } @@ -158,31 +154,46 @@ fn (mut p Page) parse_link_target(mut link Link) { parts := target.split(':') if parts.len >= 2 { link.target_collection_name = texttools.name_fix(parts[0]) - link.target_item_name = normalize_page_name(parts[1]) + // For file links, use name without extension; for page links, normalize normally + if link.is_file_link { + link.target_item_name = texttools.name_fix_no_ext(parts[1]) + } else { + link.target_item_name = normalize_page_name(parts[1]) + } } } else { - link.target_item_name = normalize_page_name(target).trim_space() + // For file links, use name without extension; for page links, normalize normally + if link.is_file_link { + link.target_item_name = texttools.name_fix_no_ext(target).trim_space() + } else { + link.target_item_name = normalize_page_name(target).trim_space() + } link.target_collection_name = p.collection.name } - if link.is_file_link == false && !p.collection.atlas.page_exists(link.key()) { - p.collection.error( - category: .invalid_page_reference - page_key: p.key() - message: 'Broken link to `${link.key()}` at line ${link.line}: `${link.src}`' - show_console: true - ) - link.status = .not_found - } else if link.is_file_link && !p.collection.atlas.file_or_image_exists(link.key()) { - p.collection.error( - category: .invalid_file_reference - page_key: p.key() - message: 'Broken file link to `${link.key()}` at line ${link.line}: `${link.src}`' - show_console: true - ) - link.status = .not_found + // Validate link target exists + mut target_exists := false + mut error_category := CollectionErrorCategory.invalid_page_reference + mut error_prefix := 'Broken link' + + if link.is_file_link { + target_exists = p.collection.atlas.file_or_image_exists(link.key()) + error_category = .invalid_file_reference + error_prefix = 'Broken file link' } else { + target_exists = p.collection.atlas.page_exists(link.key()) + } + + if target_exists { link.status = .found + } else { + p.collection.error( + category: error_category + page_key: p.key() + message: '${error_prefix} to `${link.key()}` at line ${link.line}: `${link.src}`' + show_console: true + ) + link.status = .not_found } } @@ -214,15 +225,21 @@ fn (mut p Page) content_with_fixed_links(args FixLinksArgs) !string { } // Skip cross-collection links unless enabled - if !args.cross_collection && !link.is_local_in_collection() { + is_local := link.target_collection_name == p.collection.name + if !args.cross_collection && !is_local { continue } - // Calculate new link path - new_link := p.calculate_link_path(mut link, args) or { continue } + // Calculate new link path based on mode + new_link := if args.export_mode { + p.export_link_path(mut link) or { continue } + } else { + p.filesystem_link_path(mut link) or { continue } + } // Build the complete link markdown - prefix := if link.is_file_link { '!' } else { '' } + // For image links, link.src already includes the !, so we build the same format + prefix := if link.is_image_link { '!' } else { '' } new_link_md := '${prefix}[${link.text}](${new_link})' // Replace in content @@ -232,23 +249,14 @@ fn (mut p Page) content_with_fixed_links(args FixLinksArgs) !string { return content } -// calculate_link_path returns the relative path for a link -fn (mut p Page) calculate_link_path(mut link Link, args FixLinksArgs) !string { - if args.export_mode { - // Export mode: simple flat structure - return p.export_link_path(mut link)! - } - // Fix mode: filesystem paths - return p.filesystem_link_path(mut link)! -} - // export_link_path calculates path for export (self-contained: all references are local) fn (mut p Page) export_link_path(mut link Link) !string { mut target_filename := '' if link.is_file_link { mut tf := link.target_file()! - target_filename = tf.name + // Use file_name() to include the extension + target_filename = tf.file_name() } else { mut tp := link.target_page()! target_filename = '${tp.name}.md' diff --git a/lib/web/atlas_client/client.v b/lib/web/atlas_client/client.v index feb96aff..632b34b6 100644 --- a/lib/web/atlas_client/client.v +++ b/lib/web/atlas_client/client.v @@ -388,14 +388,35 @@ pub fn (mut c AtlasClient) has_errors(collection_name string) bool { return errors.len > 0 } +// get_page_paths returns the path of a page and the paths of its linked images. +// Returns (page_path, image_paths) +// This is compatible with the doctreeclient API +pub fn (mut c AtlasClient) get_page_paths(collection_name string, page_name string) !(string, []string) { + // Get the page path + page_path := c.get_page_path(collection_name, page_name)! + page_content := c.get_page_content(collection_name, page_name)! + + // Extract image names from the page content + image_names := extract_image_links(page_content, true)! + + mut image_paths := []string{} + for image_name in image_names { + // Get the path for each image + image_path := c.get_image_path(collection_name, image_name) or { + // If an image is not found, log a warning and continue, don't fail the whole operation + return error('Error: Linked image "${image_name}" not found in collection "${collection_name}". Skipping.') + } + image_paths << image_path + } + + return page_path, image_paths +} + // copy_images copies all images linked in a page to a destination directory // This is compatible with the doctreeclient API pub fn (mut c AtlasClient) copy_images(collection_name string, page_name string, destination_path string) ! { - // Get page content - page_content := c.get_page_content(collection_name, page_name)! - - // Extract image links from content - image_names := extract_image_links(page_content, true)! + // Get the page path and linked image paths + _, image_paths := c.get_page_paths(collection_name, page_name)! // Ensure the destination directory exists os.mkdir_all(destination_path)! @@ -405,16 +426,7 @@ pub fn (mut c AtlasClient) copy_images(collection_name string, page_name string, os.mkdir_all(images_dest_path)! // Copy each linked image - for image_name in image_names { - // Get the image path - image_path := c.get_image_path(collection_name, image_name) or { - // If an image is not found, return an error - return c.error_image_not_found_linked( - collection_name: collection_name - image_name: image_name - ) - } - + for image_path in image_paths { image_file_name := os.base(image_path) dest_image_path := os.join_path(images_dest_path, image_file_name) os.cp(image_path, dest_image_path)! diff --git a/lib/web/docusaurus/config.v b/lib/web/docusaurus/config.v index 07919c27..be25188a 100644 --- a/lib/web/docusaurus/config.v +++ b/lib/web/docusaurus/config.v @@ -17,6 +17,9 @@ pub mut: reset bool template_update bool coderoot string + // Client configuration + use_atlas bool // true = atlas_client, false = doctreeclient + atlas_export_dir string // Required when use_atlas = true } @[params] @@ -28,6 +31,9 @@ pub mut: reset bool template_update bool coderoot string + // Client configuration + use_atlas bool // true = atlas_client, false = doctreeclient + atlas_export_dir string // Required when use_atlas = true } // return the last know config @@ -47,12 +53,14 @@ pub fn config() !DocusaurusConfig { } mut c := DocusaurusConfig{ - path_publish: pathlib.get_dir(path: args.path_publish, create: true)! - path_build: pathlib.get_dir(path: args.path_build, create: true)! - coderoot: args.coderoot - install: args.install - reset: args.reset - template_update: args.template_update + path_publish: pathlib.get_dir(path: args.path_publish, create: true)! + path_build: pathlib.get_dir(path: args.path_build, create: true)! + coderoot: args.coderoot + install: args.install + reset: args.reset + template_update: args.template_update + use_atlas: args.use_atlas + atlas_export_dir: args.atlas_export_dir } if c.install { install(c)! diff --git a/lib/web/docusaurus/dsite_generate_docs.v b/lib/web/docusaurus/dsite_generate_docs.v index 34749165..91b76ba2 100644 --- a/lib/web/docusaurus/dsite_generate_docs.v +++ b/lib/web/docusaurus/dsite_generate_docs.v @@ -1,18 +1,53 @@ module docusaurus import incubaid.herolib.core.pathlib +import incubaid.herolib.web.atlas_client import incubaid.herolib.web.doctreeclient import incubaid.herolib.web.site { Page, Section, Site } import incubaid.herolib.data.markdown.tools as markdowntools import incubaid.herolib.ui.console -// THIS CODE GENERATES A DOCUSAURUS SITE FROM A DOCTREECLIENT AND SITE DEFINITION +// THIS CODE GENERATES A DOCUSAURUS SITE FROM A DOCUMENT CLIENT AND SITE DEFINITION +// Supports both atlas_client and doctreeclient through the unified IDocClient interface + +// IDocClient defines the common interface that both atlas_client and doctreeclient implement +// This allows the Docusaurus module to work with either client transparently +// +// Note: V interfaces require exact signature matching, so all methods use `mut` receivers +// to match the implementation in both atlas_client and doctreeclient +pub interface IDocClient { +mut: + // Path methods - get absolute paths to resources + get_page_path(collection_name string, page_name string) !string + get_file_path(collection_name string, file_name string) !string + get_image_path(collection_name string, image_name string) !string + + // Existence checks - verify if resources exist + page_exists(collection_name string, page_name string) bool + file_exists(collection_name string, file_name string) bool + image_exists(collection_name string, image_name string) bool + + // Content retrieval + get_page_content(collection_name string, page_name string) !string + + // Listing methods - enumerate resources + list_collections() ![]string + list_pages(collection_name string) ![]string + list_files(collection_name string) ![]string + list_images(collection_name string) ![]string + list_pages_map() !map[string][]string + list_markdown() !string + + // Image operations + get_page_paths(collection_name string, page_name string) !(string, []string) + copy_images(collection_name string, page_name string, destination_path string) ! +} struct SiteGenerator { mut: siteconfig_name string path pathlib.Path - client &doctreeclient.DocTreeClient + client IDocClient flat bool // if flat then won't use sitenames as subdir's site Site errors []string // collect errors here @@ -25,9 +60,21 @@ pub fn (mut docsite DocSite) generate_docs() ! { // we generate the docs in the build path docs_path := '${c.path_build.path}/docs' + // Create the appropriate client based on configuration + mut client := if c.use_atlas { + // Use atlas_client for filesystem-based access + if c.atlas_export_dir == '' { + return error('atlas_export_dir is required when use_atlas is true') + } + IDocClient(atlas_client.new(export_dir: c.atlas_export_dir)!) + } else { + // Use doctreeclient for Redis-based access + IDocClient(doctreeclient.new()!) + } + mut gen := SiteGenerator{ path: pathlib.get_dir(path: docs_path, create: true)! - client: doctreeclient.new()! + client: client flat: true site: docsite.website } @@ -368,8 +415,65 @@ fn (generator SiteGenerator) fix_links(content string, current_page_path string) } } - // STEP 5: Remove .md extensions from all links (Docusaurus doesn't use them in URLs) + // STEP 5: Fix bare page references (from atlas self-contained exports) + // Atlas exports convert cross-collection links to simple relative links like "token_system2.md" + // We need to transform these to proper relative paths based on Docusaurus structure + for page_name, target_dir in page_to_path { + // Match links in the format ](page_name) or ](page_name.md) + old_link_with_md := '](${page_name}.md)' + old_link_without_md := '](${page_name})' + + if result.contains(old_link_with_md) || result.contains(old_link_without_md) { + new_link := calculate_relative_path(current_dir, target_dir, page_name) + // Replace both .md and non-.md versions + result = result.replace(old_link_with_md, '](${new_link})') + result = result.replace(old_link_without_md, '](${new_link})') + } + } + + // STEP 6: Remove .md extensions from all remaining links (Docusaurus doesn't use them in URLs) result = result.replace('.md)', ')') + // STEP 7: Fix image links to point to img/ subdirectory + // Images are copied to img/ subdirectory by copy_images(), so we need to update the links + // Transform ![alt](image.png) to ![alt](img/image.png) for local images only + mut image_lines := result.split('\n') + for i, line in image_lines { + // Find image links: ![...](...) but skip external URLs + if line.contains('![') { + mut pos := 0 + for { + img_start := line.index_after('![', pos) or { break } + alt_end := line.index_after(']', img_start) or { break } + if alt_end + 1 >= line.len || line[alt_end + 1] != `(` { + pos = alt_end + 1 + continue + } + url_start := alt_end + 2 + url_end := line.index_after(')', url_start) or { break } + url := line[url_start..url_end] + + // Skip external URLs and already-prefixed img/ paths + if url.starts_with('http://') || url.starts_with('https://') + || url.starts_with('img/') || url.starts_with('./img/') { + pos = url_end + 1 + continue + } + + // Skip absolute paths and paths with ../ + if url.starts_with('/') || url.starts_with('../') { + pos = url_end + 1 + continue + } + + // This is a local image reference - add img/ prefix + new_url := 'img/${url}' + image_lines[i] = line[0..url_start] + new_url + line[url_end..] + break + } + } + } + result = image_lines.join('\n') + return result } diff --git a/lib/web/docusaurus/play.v b/lib/web/docusaurus/play.v index 5e7bbeb3..23aed3ca 100644 --- a/lib/web/docusaurus/play.v +++ b/lib/web/docusaurus/play.v @@ -14,11 +14,13 @@ pub fn play(mut plbook PlayBook) ! { mut param_define := action_define.params config_set( - path_build: param_define.get_default('path_build', '')! - path_publish: param_define.get_default('path_publish', '')! - reset: param_define.get_default_false('reset') - template_update: param_define.get_default_false('template_update') - install: param_define.get_default_false('install') + path_build: param_define.get_default('path_build', '')! + path_publish: param_define.get_default('path_publish', '')! + reset: param_define.get_default_false('reset') + template_update: param_define.get_default_false('template_update') + install: param_define.get_default_false('install') + use_atlas: param_define.get_default_false('use_atlas') + atlas_export_dir: param_define.get_default('atlas_export_dir', '')! )! site_name := param_define.get('name') or {