feat: Enhance docusaurus site generation with atlas client

- Add flags for development server and browser opening
- Introduce IDocClient interface for unified client access
- Implement atlas_client integration for Docusaurus
- Refactor link handling and image path resolution
- Update Docusaurus config with atlas client options
This commit is contained in:
Mahmoud-Emad
2025-11-06 15:44:09 +02:00
parent 5fccd03ee7
commit a149845fc7
8 changed files with 303 additions and 96 deletions

View File

@@ -4,6 +4,7 @@ import incubaid.herolib.ui.console
import incubaid.herolib.data.atlas
import incubaid.herolib.core.playcmds
import incubaid.herolib.develop.gittools
import incubaid.herolib.web.docusaurus
import os
import cli { Command, Flag }
@@ -92,6 +93,21 @@ pub fn cmd_atlas(mut cmdroot Command) Command {
description: 'Update environment and git pull before operations.'
})
cmd_run.add_flag(Flag{
flag: .bool
required: false
name: 'dev'
description: 'Run development server after export (requires docusaurus config).'
})
cmd_run.add_flag(Flag{
flag: .bool
required: false
name: 'open'
abbrev: 'o'
description: 'Open browser when running dev server (use with --dev).'
})
cmdroot.add_command(cmd_run)
return cmdroot
}
@@ -102,6 +118,8 @@ fn cmd_atlas_execute(cmd Command) ! {
mut update := cmd.flags.get_bool('update') or { false }
mut scan := cmd.flags.get_bool('scan') or { false }
mut export := cmd.flags.get_bool('export') or { false }
mut dev := cmd.flags.get_bool('dev') or { false }
mut open_ := cmd.flags.get_bool('open') or { false }
// Include and redis default to true unless explicitly disabled
mut no_include := cmd.flags.get_bool('no-include') or { false }
@@ -132,7 +150,7 @@ fn cmd_atlas_execute(cmd Command) ! {
// Run HeroScript if exists
playcmds.run(
heroscript_path: atlas_path.path
reset: false
reset: reset
emptycheck: false
)!
@@ -181,5 +199,27 @@ fn cmd_atlas_execute(cmd Command) ! {
col.print_errors()
}
}
// Run dev server if -dev flag is set
if dev {
console.print_header('Starting development server...')
console.print_item('Atlas export directory: ${destination}')
console.print_item('Looking for docusaurus configuration in: ${atlas_path.path}')
// Run the docusaurus dev server using the exported atlas content
// This will look for a .heroscript file in the atlas_path that configures docusaurus
// with use_atlas:true and atlas_export_dir pointing to the destination
playcmds.run(
heroscript_path: atlas_path.path
reset: reset
)!
// Get the docusaurus site and run dev server
mut dsite := docusaurus.dsite_get('')!
dsite.dev(
open: open_
watch_changes: false
)!
}
}
}

View File

@@ -70,9 +70,10 @@ fn (mut c Collection) add_page(mut path pathlib.Path) ! {
// Add an image to the collection
fn (mut c Collection) add_file(mut p pathlib.Path) ! {
name := p.name_fix_keepext()
// Use name without extension for the key and name field
name := p.name_fix_no_ext()
if name in c.files {
return error('Page ${name} already exists in collection ${c.name}')
return error('File ${name} already exists in collection ${c.name}')
}
// Use absolute paths for path_relative to work correctly
mut col_path := pathlib.get(c.path)
@@ -80,7 +81,7 @@ fn (mut c Collection) add_file(mut p pathlib.Path) ! {
relativepath := file_abs_path.path_relative(col_path.absolute())!
mut file_new := File{
name: name
name: name // name without extension
ext: p.extension_lower()
path: relativepath // relative path of file in the collection
collection: &c

View File

@@ -67,8 +67,9 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! {
)!
json_file.write(meta)!
// Track cross-collection pages that need to be copied for self-contained export
// Track cross-collection pages and files that need to be copied for self-contained export
mut cross_collection_pages := map[string]&Page{} // key: page.name, value: &Page
mut cross_collection_files := map[string]&File{} // key: file.name, value: &File
// First pass: export all pages in this collection and collect cross-collection references
for _, mut page in c.pages {
@@ -82,17 +83,32 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! {
mut dest_file := pathlib.get_file(path: '${col_dir.path}/${page.name}.md', create: true)!
dest_file.write(content)!
// Collect cross-collection page references for copying
// Collect cross-collection references for copying (pages and files/images)
// IMPORTANT: Use cached links from validation (before transformation) to preserve collection info
for mut link in page.links {
// Only process valid page links (not files/images) from other collections
if link.status == .found && !link.is_file_link && !link.is_local_in_collection() {
if link.status != .found {
continue
}
// Collect cross-collection page references
is_local := link.target_collection_name == c.name
if !link.is_file_link && !is_local {
mut target_page := link.target_page() or { continue }
// Use page name as key to avoid duplicates
if target_page.name !in cross_collection_pages {
cross_collection_pages[target_page.name] = target_page
}
}
// Collect cross-collection file/image references
if link.is_file_link && !is_local {
mut target_file := link.target_file() or { continue }
// Use file name as key to avoid duplicates
file_key := target_file.file_name()
if file_key !in cross_collection_files {
cross_collection_files[file_key] = target_file
}
}
}
// Redis operations...
@@ -103,6 +119,14 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! {
}
}
// Copy all files/images from this collection to the export directory
for _, mut file in c.files {
mut src_file := file.path()!
mut dest_path := '${col_dir.path}/${file.file_name()}'
mut dest_file := pathlib.get_file(path: dest_path, create: true)!
src_file.copy(dest: dest_file.path)!
}
// Second pass: copy cross-collection referenced pages to make collection self-contained
for _, mut ref_page in cross_collection_pages {
// Get the referenced page content with includes processed
@@ -116,4 +140,12 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! {
mut dest_file := pathlib.get_file(path: '${col_dir.path}/${ref_page.name}.md', create: true)!
dest_file.write(ref_content)!
}
// Third pass: copy cross-collection referenced files/images to make collection self-contained
for _, mut ref_file in cross_collection_files {
mut src_file := ref_file.path()!
mut dest_path := '${col_dir.path}/${ref_file.file_name()}'
mut dest_file := pathlib.get_file(path: dest_path, create: true)!
src_file.copy(dest: dest_file.path)!
}
}

View File

@@ -27,20 +27,12 @@ pub enum LinkStatus {
error
}
// Get the collection:item key for this link
fn (mut self Link) key() string {
return '${self.target_collection_name}:${self.target_item_name}'
}
// is the link in the same collection as the page containing the link
fn (mut self Link) is_local_in_collection() bool {
return self.target_collection_name == self.page.collection.name
}
// is the link pointing to an external resource e.g. http, git, mailto, ftp
pub fn (mut self Link) is_external() bool {
return self.status == .external
}
// Get the target page this link points to
pub fn (mut self Link) target_page() !&Page {
if self.status == .external {
return error('External links do not have a target page')
@@ -48,6 +40,7 @@ pub fn (mut self Link) target_page() !&Page {
return self.page.collection.atlas.page_get(self.key())
}
// Get the target file this link points to
pub fn (mut self Link) target_file() !&File {
if self.status == .external {
return error('External links do not have a target file')
@@ -92,21 +85,23 @@ fn (mut p Page) find_links(content string) ![]Link {
text := line[open_bracket + 1..close_bracket]
target := line[open_paren + 1..close_paren]
mut is_image_link := (image_open != -1)
mut is_file_link := false
// if no . in file then it means it's a page link (binaries with . are not supported in other words)
if target.contains('.') && (!target.trim_space().to_lower().ends_with('.md')) {
is_file_link = true
is_image_link = false // means it's a file link, not an image link
}
// Determine link type
// File links have extensions (but not .md), image links start with ![
is_file_link := target.contains('.') && !target.trim_space().to_lower().ends_with('.md')
is_image_link := image_open != -1 && !is_file_link
// Store position - use image_open if it's an image, otherwise open_bracket
link_start_pos := if is_image_link { image_open } else { open_bracket }
// For image links, src should include the ! prefix
link_src := if is_image_link {
line[image_open..close_paren + 1]
} else {
line[open_bracket..close_paren + 1]
}
mut link := Link{
src: line[open_bracket..close_paren + 1]
src: link_src
text: text
target: target.trim_space()
line: line_idx + 1
@@ -133,23 +128,24 @@ fn (mut p Page) find_links(content string) ![]Link {
fn (mut p Page) parse_link_target(mut link Link) {
mut target := link.target.to_lower().trim_space()
// Skip external links
// Check for external links (http, https, mailto, ftp)
if target.starts_with('http://') || target.starts_with('https://')
|| target.starts_with('mailto:') || target.starts_with('ftp://') {
link.status = .external
return
}
// Skip anchors
// Check for anchor links
if target.starts_with('#') {
link.status = .anchor
return
}
// Handle relative paths - extract the last part after /
if target.contains('/') {
parts9 := target.split('/')
if parts9.len >= 1 {
target = parts9[1]
parts := target.split('/')
if parts.len > 1 {
target = parts[parts.len - 1]
}
}
@@ -158,31 +154,46 @@ fn (mut p Page) parse_link_target(mut link Link) {
parts := target.split(':')
if parts.len >= 2 {
link.target_collection_name = texttools.name_fix(parts[0])
link.target_item_name = normalize_page_name(parts[1])
// For file links, use name without extension; for page links, normalize normally
if link.is_file_link {
link.target_item_name = texttools.name_fix_no_ext(parts[1])
} else {
link.target_item_name = normalize_page_name(parts[1])
}
}
} else {
link.target_item_name = normalize_page_name(target).trim_space()
// For file links, use name without extension; for page links, normalize normally
if link.is_file_link {
link.target_item_name = texttools.name_fix_no_ext(target).trim_space()
} else {
link.target_item_name = normalize_page_name(target).trim_space()
}
link.target_collection_name = p.collection.name
}
if link.is_file_link == false && !p.collection.atlas.page_exists(link.key()) {
p.collection.error(
category: .invalid_page_reference
page_key: p.key()
message: 'Broken link to `${link.key()}` at line ${link.line}: `${link.src}`'
show_console: true
)
link.status = .not_found
} else if link.is_file_link && !p.collection.atlas.file_or_image_exists(link.key()) {
p.collection.error(
category: .invalid_file_reference
page_key: p.key()
message: 'Broken file link to `${link.key()}` at line ${link.line}: `${link.src}`'
show_console: true
)
link.status = .not_found
// Validate link target exists
mut target_exists := false
mut error_category := CollectionErrorCategory.invalid_page_reference
mut error_prefix := 'Broken link'
if link.is_file_link {
target_exists = p.collection.atlas.file_or_image_exists(link.key())
error_category = .invalid_file_reference
error_prefix = 'Broken file link'
} else {
target_exists = p.collection.atlas.page_exists(link.key())
}
if target_exists {
link.status = .found
} else {
p.collection.error(
category: error_category
page_key: p.key()
message: '${error_prefix} to `${link.key()}` at line ${link.line}: `${link.src}`'
show_console: true
)
link.status = .not_found
}
}
@@ -214,15 +225,21 @@ fn (mut p Page) content_with_fixed_links(args FixLinksArgs) !string {
}
// Skip cross-collection links unless enabled
if !args.cross_collection && !link.is_local_in_collection() {
is_local := link.target_collection_name == p.collection.name
if !args.cross_collection && !is_local {
continue
}
// Calculate new link path
new_link := p.calculate_link_path(mut link, args) or { continue }
// Calculate new link path based on mode
new_link := if args.export_mode {
p.export_link_path(mut link) or { continue }
} else {
p.filesystem_link_path(mut link) or { continue }
}
// Build the complete link markdown
prefix := if link.is_file_link { '!' } else { '' }
// For image links, link.src already includes the !, so we build the same format
prefix := if link.is_image_link { '!' } else { '' }
new_link_md := '${prefix}[${link.text}](${new_link})'
// Replace in content
@@ -232,23 +249,14 @@ fn (mut p Page) content_with_fixed_links(args FixLinksArgs) !string {
return content
}
// calculate_link_path returns the relative path for a link
fn (mut p Page) calculate_link_path(mut link Link, args FixLinksArgs) !string {
if args.export_mode {
// Export mode: simple flat structure
return p.export_link_path(mut link)!
}
// Fix mode: filesystem paths
return p.filesystem_link_path(mut link)!
}
// export_link_path calculates path for export (self-contained: all references are local)
fn (mut p Page) export_link_path(mut link Link) !string {
mut target_filename := ''
if link.is_file_link {
mut tf := link.target_file()!
target_filename = tf.name
// Use file_name() to include the extension
target_filename = tf.file_name()
} else {
mut tp := link.target_page()!
target_filename = '${tp.name}.md'

View File

@@ -388,14 +388,35 @@ pub fn (mut c AtlasClient) has_errors(collection_name string) bool {
return errors.len > 0
}
// get_page_paths returns the path of a page and the paths of its linked images.
// Returns (page_path, image_paths)
// This is compatible with the doctreeclient API
pub fn (mut c AtlasClient) get_page_paths(collection_name string, page_name string) !(string, []string) {
// Get the page path
page_path := c.get_page_path(collection_name, page_name)!
page_content := c.get_page_content(collection_name, page_name)!
// Extract image names from the page content
image_names := extract_image_links(page_content, true)!
mut image_paths := []string{}
for image_name in image_names {
// Get the path for each image
image_path := c.get_image_path(collection_name, image_name) or {
// If an image is not found, log a warning and continue, don't fail the whole operation
return error('Error: Linked image "${image_name}" not found in collection "${collection_name}". Skipping.')
}
image_paths << image_path
}
return page_path, image_paths
}
// copy_images copies all images linked in a page to a destination directory
// This is compatible with the doctreeclient API
pub fn (mut c AtlasClient) copy_images(collection_name string, page_name string, destination_path string) ! {
// Get page content
page_content := c.get_page_content(collection_name, page_name)!
// Extract image links from content
image_names := extract_image_links(page_content, true)!
// Get the page path and linked image paths
_, image_paths := c.get_page_paths(collection_name, page_name)!
// Ensure the destination directory exists
os.mkdir_all(destination_path)!
@@ -405,16 +426,7 @@ pub fn (mut c AtlasClient) copy_images(collection_name string, page_name string,
os.mkdir_all(images_dest_path)!
// Copy each linked image
for image_name in image_names {
// Get the image path
image_path := c.get_image_path(collection_name, image_name) or {
// If an image is not found, return an error
return c.error_image_not_found_linked(
collection_name: collection_name
image_name: image_name
)
}
for image_path in image_paths {
image_file_name := os.base(image_path)
dest_image_path := os.join_path(images_dest_path, image_file_name)
os.cp(image_path, dest_image_path)!

View File

@@ -17,6 +17,9 @@ pub mut:
reset bool
template_update bool
coderoot string
// Client configuration
use_atlas bool // true = atlas_client, false = doctreeclient
atlas_export_dir string // Required when use_atlas = true
}
@[params]
@@ -28,6 +31,9 @@ pub mut:
reset bool
template_update bool
coderoot string
// Client configuration
use_atlas bool // true = atlas_client, false = doctreeclient
atlas_export_dir string // Required when use_atlas = true
}
// return the last know config
@@ -47,12 +53,14 @@ pub fn config() !DocusaurusConfig {
}
mut c := DocusaurusConfig{
path_publish: pathlib.get_dir(path: args.path_publish, create: true)!
path_build: pathlib.get_dir(path: args.path_build, create: true)!
coderoot: args.coderoot
install: args.install
reset: args.reset
template_update: args.template_update
path_publish: pathlib.get_dir(path: args.path_publish, create: true)!
path_build: pathlib.get_dir(path: args.path_build, create: true)!
coderoot: args.coderoot
install: args.install
reset: args.reset
template_update: args.template_update
use_atlas: args.use_atlas
atlas_export_dir: args.atlas_export_dir
}
if c.install {
install(c)!

View File

@@ -1,18 +1,53 @@
module docusaurus
import incubaid.herolib.core.pathlib
import incubaid.herolib.web.atlas_client
import incubaid.herolib.web.doctreeclient
import incubaid.herolib.web.site { Page, Section, Site }
import incubaid.herolib.data.markdown.tools as markdowntools
import incubaid.herolib.ui.console
// THIS CODE GENERATES A DOCUSAURUS SITE FROM A DOCTREECLIENT AND SITE DEFINITION
// THIS CODE GENERATES A DOCUSAURUS SITE FROM A DOCUMENT CLIENT AND SITE DEFINITION
// Supports both atlas_client and doctreeclient through the unified IDocClient interface
// IDocClient defines the common interface that both atlas_client and doctreeclient implement
// This allows the Docusaurus module to work with either client transparently
//
// Note: V interfaces require exact signature matching, so all methods use `mut` receivers
// to match the implementation in both atlas_client and doctreeclient
pub interface IDocClient {
mut:
// Path methods - get absolute paths to resources
get_page_path(collection_name string, page_name string) !string
get_file_path(collection_name string, file_name string) !string
get_image_path(collection_name string, image_name string) !string
// Existence checks - verify if resources exist
page_exists(collection_name string, page_name string) bool
file_exists(collection_name string, file_name string) bool
image_exists(collection_name string, image_name string) bool
// Content retrieval
get_page_content(collection_name string, page_name string) !string
// Listing methods - enumerate resources
list_collections() ![]string
list_pages(collection_name string) ![]string
list_files(collection_name string) ![]string
list_images(collection_name string) ![]string
list_pages_map() !map[string][]string
list_markdown() !string
// Image operations
get_page_paths(collection_name string, page_name string) !(string, []string)
copy_images(collection_name string, page_name string, destination_path string) !
}
struct SiteGenerator {
mut:
siteconfig_name string
path pathlib.Path
client &doctreeclient.DocTreeClient
client IDocClient
flat bool // if flat then won't use sitenames as subdir's
site Site
errors []string // collect errors here
@@ -25,9 +60,21 @@ pub fn (mut docsite DocSite) generate_docs() ! {
// we generate the docs in the build path
docs_path := '${c.path_build.path}/docs'
// Create the appropriate client based on configuration
mut client := if c.use_atlas {
// Use atlas_client for filesystem-based access
if c.atlas_export_dir == '' {
return error('atlas_export_dir is required when use_atlas is true')
}
IDocClient(atlas_client.new(export_dir: c.atlas_export_dir)!)
} else {
// Use doctreeclient for Redis-based access
IDocClient(doctreeclient.new()!)
}
mut gen := SiteGenerator{
path: pathlib.get_dir(path: docs_path, create: true)!
client: doctreeclient.new()!
client: client
flat: true
site: docsite.website
}
@@ -368,8 +415,65 @@ fn (generator SiteGenerator) fix_links(content string, current_page_path string)
}
}
// STEP 5: Remove .md extensions from all links (Docusaurus doesn't use them in URLs)
// STEP 5: Fix bare page references (from atlas self-contained exports)
// Atlas exports convert cross-collection links to simple relative links like "token_system2.md"
// We need to transform these to proper relative paths based on Docusaurus structure
for page_name, target_dir in page_to_path {
// Match links in the format ](page_name) or ](page_name.md)
old_link_with_md := '](${page_name}.md)'
old_link_without_md := '](${page_name})'
if result.contains(old_link_with_md) || result.contains(old_link_without_md) {
new_link := calculate_relative_path(current_dir, target_dir, page_name)
// Replace both .md and non-.md versions
result = result.replace(old_link_with_md, '](${new_link})')
result = result.replace(old_link_without_md, '](${new_link})')
}
}
// STEP 6: Remove .md extensions from all remaining links (Docusaurus doesn't use them in URLs)
result = result.replace('.md)', ')')
// STEP 7: Fix image links to point to img/ subdirectory
// Images are copied to img/ subdirectory by copy_images(), so we need to update the links
// Transform ![alt](image.png) to ![alt](img/image.png) for local images only
mut image_lines := result.split('\n')
for i, line in image_lines {
// Find image links: ![...](...) but skip external URLs
if line.contains('![') {
mut pos := 0
for {
img_start := line.index_after('![', pos) or { break }
alt_end := line.index_after(']', img_start) or { break }
if alt_end + 1 >= line.len || line[alt_end + 1] != `(` {
pos = alt_end + 1
continue
}
url_start := alt_end + 2
url_end := line.index_after(')', url_start) or { break }
url := line[url_start..url_end]
// Skip external URLs and already-prefixed img/ paths
if url.starts_with('http://') || url.starts_with('https://')
|| url.starts_with('img/') || url.starts_with('./img/') {
pos = url_end + 1
continue
}
// Skip absolute paths and paths with ../
if url.starts_with('/') || url.starts_with('../') {
pos = url_end + 1
continue
}
// This is a local image reference - add img/ prefix
new_url := 'img/${url}'
image_lines[i] = line[0..url_start] + new_url + line[url_end..]
break
}
}
}
result = image_lines.join('\n')
return result
}

View File

@@ -14,11 +14,13 @@ pub fn play(mut plbook PlayBook) ! {
mut param_define := action_define.params
config_set(
path_build: param_define.get_default('path_build', '')!
path_publish: param_define.get_default('path_publish', '')!
reset: param_define.get_default_false('reset')
template_update: param_define.get_default_false('template_update')
install: param_define.get_default_false('install')
path_build: param_define.get_default('path_build', '')!
path_publish: param_define.get_default('path_publish', '')!
reset: param_define.get_default_false('reset')
template_update: param_define.get_default_false('template_update')
install: param_define.get_default_false('install')
use_atlas: param_define.get_default_false('use_atlas')
atlas_export_dir: param_define.get_default('atlas_export_dir', '')!
)!
site_name := param_define.get('name') or {