diff --git a/examples/data/atlas/debug_recursive/debug_atlas.vsh b/examples/data/atlas/debug_recursive/debug_atlas.vsh new file mode 100644 index 00000000..bb6a0847 --- /dev/null +++ b/examples/data/atlas/debug_recursive/debug_atlas.vsh @@ -0,0 +1,308 @@ +#!/usr/bin/env -S vrun + +import incubaid.herolib.data.atlas +import incubaid.herolib.ui.console +import os + +fn main() { + println('=== ATLAS DEBUG SCRIPT ===\n') + + // Create and scan atlas + mut a := atlas.new(name: 'main')! + + // Scan the collections + println('Scanning collections...\n') + a.scan( + path: '/Users/despiegk/code/git.ourworld.tf/geomind/docs_geomind/collections/mycelium_nodes_tiers' + )! + a.scan( + path: '/Users/despiegk/code/git.ourworld.tf/geomind/docs_geomind/collections/geomind_compare' + )! + a.scan(path: '/Users/despiegk/code/git.ourworld.tf/geomind/docs_geomind/collections/geoaware')! + a.scan( + path: '/Users/despiegk/code/git.ourworld.tf/tfgrid/docs_tfgrid4/collections/mycelium_economics' + )! + a.scan( + path: '/Users/despiegk/code/git.ourworld.tf/tfgrid/docs_tfgrid4/collections/mycelium_concepts' + )! + a.scan( + path: '/Users/despiegk/code/git.ourworld.tf/tfgrid/docs_tfgrid4/collections/mycelium_cloud_tech' + )! + + // Initialize atlas (post-scanning validation) + a.init_post()! + + // Print all pages per collection + println('\n=== COLLECTIONS & PAGES ===\n') + for col_name, col in a.collections { + println('Collection: ${col_name}') + println(' Pages (${col.pages.len}):') + if col.pages.len > 0 { + for page_name, _ in col.pages { + println(' - ${page_name}') + } + } else { + println(' (empty)') + } + println(' Files/Images (${col.files.len}):') + if col.files.len > 0 { + for file_name, _ in col.files { + println(' - ${file_name}') + } + } else { + println(' (empty)') + } + } + + // Validate links (this will recursively find links across collections) + println('\n=== VALIDATING LINKS (RECURSIVE) ===\n') + a.validate_links()! + println('✓ Link validation complete\n') + + // Check for broken links + println('\n=== BROKEN LINKS ===\n') + mut total_errors := 0 + for col_name, col in a.collections { + if col.has_errors() { + println('Collection: ${col_name} (${col.errors.len} errors)') + for err in col.errors { + println(' [${err.category_str()}] Page: ${err.page_key}') + println(' Message: ${err.message}') + println('') + total_errors++ + } + } + } + + if total_errors == 0 { + println('✓ No broken links found!') + } else { + println('\n❌ Total broken link errors: ${total_errors}') + } + + // Show discovered links per page (validates recursive discovery) + println('\n\n=== DISCOVERED LINKS (RECURSIVE RESOLUTION) ===\n') + println('Checking for files referenced by cross-collection pages...\n') + mut total_links := 0 + for col_name, col in a.collections { + mut col_has_links := false + for page_name, page in col.pages { + if page.links.len > 0 { + if !col_has_links { + println('Collection: ${col_name}') + col_has_links = true + } + println(' Page: ${page_name} (${page.links.len} links)') + for link in page.links { + target_col := if link.target_collection_name != '' { + link.target_collection_name + } else { + col_name + } + println(' → ${target_col}:${link.target_item_name} [${link.file_type}]') + total_links++ + } + } + } + } + println('\n✓ Total links discovered: ${total_links}') + + // List pages that need investigation + println('\n=== CHECKING SPECIFIC MISSING PAGES ===\n') + + missing_pages := [ + 'compare_electricity', + 'internet_basics', + 'centralization_risk', + 'gdp_negative', + ] + + // Check in geoaware collection + if 'geoaware' in a.collections { + mut geoaware := a.get_collection('geoaware')! + + println('Collection: geoaware') + if geoaware.pages.len > 0 { + println(' All pages in collection:') + for page_name, _ in geoaware.pages { + println(' - ${page_name}') + } + } else { + println(' (No pages found)') + } + + println('\n Checking for specific missing pages:') + for page_name in missing_pages { + exists := page_name in geoaware.pages + status := if exists { '✓' } else { '✗' } + println(' ${status} ${page_name}') + } + } + + // Check for pages across all collections + println('\n\n=== LOOKING FOR MISSING PAGES ACROSS ALL COLLECTIONS ===\n') + + for missing_page in missing_pages { + println('Searching for "${missing_page}":') + mut found := false + for col_name, col in a.collections { + if missing_page in col.pages { + println(' ✓ Found in: ${col_name}') + found = true + } + } + if !found { + println(' ✗ Not found in any collection') + } + } + + // Check for the solution page + println('\n\n=== CHECKING FOR "solution" PAGE ===\n') + for col_name in ['mycelium_nodes_tiers', 'geomind_compare', 'geoaware', 'mycelium_economics', + 'mycelium_concepts', 'mycelium_cloud_tech'] { + if col_name in a.collections { + mut col := a.get_collection(col_name)! + exists := col.page_exists('solution')! + status := if exists { '✓' } else { '✗' } + println('${status} ${col_name}: "solution" page') + } + } + + // Print error summary + println('\n\n=== ERROR SUMMARY BY CATEGORY ===\n') + mut category_counts := map[string]int{} + for _, col in a.collections { + for err in col.errors { + cat_str := err.category_str() + category_counts[cat_str]++ + } + } + + if category_counts.len == 0 { + println('✓ No errors found!') + } else { + for cat, count in category_counts { + println('${cat}: ${count}') + } + } + + // ===== EXPORT AND FILE VERIFICATION TEST ===== + println('\n\n=== EXPORT AND FILE VERIFICATION TEST ===\n') + + // Create export directory + export_path := '/tmp/atlas_debug_export' + if os.exists(export_path) { + os.rmdir_all(export_path)! + } + os.mkdir_all(export_path)! + + println('Exporting to: ${export_path}\n') + a.export(destination: export_path)! + println('✓ Export completed\n') + + // Collect all files found during link validation + mut expected_files := map[string]string{} // key: file_name, value: collection_name + mut file_count := 0 + for col_name, col in a.collections { + for page_name, page in col.pages { + for link in page.links { + if link.status == .found && (link.file_type == .file || link.file_type == .image) { + file_key := link.target_item_name + expected_files[file_key] = link.target_collection_name + file_count++ + } + } + } + } + + println('Expected to find ${file_count} file references in links\n') + println('=== VERIFYING FILES IN EXPORT DIRECTORY ===\n') + + // Get the first collection name (the primary exported collection) + mut primary_col_name := '' + for col_name, _ in a.collections { + primary_col_name = col_name + break + } + + if primary_col_name == '' { + println('❌ No collections found') + } else { + mut verified_count := 0 + mut missing_count := 0 + mut found_files := map[string]bool{} + + // Check both img and files directories + img_dir := '${export_path}/content/${primary_col_name}/img' + files_dir := '${export_path}/content/${primary_col_name}/files' + + // Scan img directory + if os.exists(img_dir) { + img_files := os.ls(img_dir) or { []string{} } + for img_file in img_files { + found_files[img_file] = true + } + } + + // Scan files directory + if os.exists(files_dir) { + file_list := os.ls(files_dir) or { []string{} } + for file in file_list { + found_files[file] = true + } + } + + println('Files/Images found in export directory:') + if found_files.len > 0 { + for file_name, _ in found_files { + println(' ✓ ${file_name}') + if file_name in expected_files { + verified_count++ + } + } + } else { + println(' (none found)') + } + + println('\n=== FILE VERIFICATION RESULTS ===\n') + println('Expected files from links: ${file_count}') + println('Files found in export: ${found_files.len}') + println('Files verified (present in export): ${verified_count}') + + // Check for missing expected files + for expected_file, source_col in expected_files { + if expected_file !in found_files { + missing_count++ + println(' ✗ Missing: ${expected_file} (from ${source_col})') + } + } + + if missing_count > 0 { + println('\n❌ ${missing_count} expected files are MISSING from export!') + } else if verified_count == file_count && file_count > 0 { + println('\n✓ All expected files are present in export directory!') + } else if file_count == 0 { + println('\n⚠ No file links were found during validation (check if pages have file references)') + } + + // Show directory structure + println('\n=== EXPORT DIRECTORY STRUCTURE ===\n') + if os.exists('${export_path}/content/${primary_col_name}') { + println('${export_path}/content/${primary_col_name}/') + + content_files := os.ls('${export_path}/content/${primary_col_name}') or { []string{} } + for item in content_files { + full_path := '${export_path}/content/${primary_col_name}/${item}' + if os.is_dir(full_path) { + sub_items := os.ls(full_path) or { []string{} } + println(' ${item}/ (${sub_items.len} items)') + for sub_item in sub_items { + println(' - ${sub_item}') + } + } else { + println(' - ${item}') + } + } + } + } +} diff --git a/lib/data/atlas/client/README.md b/lib/data/atlas/client/README.md index 6d3d79b3..f588640f 100644 --- a/lib/data/atlas/client/README.md +++ b/lib/data/atlas/client/README.md @@ -17,8 +17,8 @@ AtlasClient provides methods to: ```v import incubaid.herolib.web.atlas_client -// Create client -mut client := atlas_client.new(export_dir: '/tmp/atlas_export')! +// Create client, exports will be in $/hero/var/atlas_export by default +mut client := atlas_client.new()! // List collections collections := client.list_collections()! diff --git a/lib/data/atlas/client/client.v b/lib/data/atlas/client/client.v index 01140d90..df066217 100644 --- a/lib/data/atlas/client/client.v +++ b/lib/data/atlas/client/client.v @@ -247,20 +247,6 @@ pub fn (mut c AtlasClient) get_collection_metadata(collection_name string) !Coll return metadata } -// get_page_links returns the links found in a page by reading the metadata -pub fn (mut c AtlasClient) get_page_links(collection_name string, page_name string) ![]LinkMetadata { - // Get collection metadata - metadata := c.get_collection_metadata(collection_name)! - // Apply name normalization to page name - fixed_page_name := texttools.name_fix_no_ext(page_name) - - // Find the page in metadata - if fixed_page_name in metadata.pages { - return metadata.pages[fixed_page_name].links - } - return error('page_not_found: Page "${page_name}" not found in collection metadata, for collection: "${collection_name}"') -} - // get_collection_errors returns the errors for a collection from metadata pub fn (mut c AtlasClient) get_collection_errors(collection_name string) ![]ErrorMetadata { metadata := c.get_collection_metadata(collection_name)! @@ -273,6 +259,30 @@ pub fn (mut c AtlasClient) has_errors(collection_name string) bool { return errors.len > 0 } +pub fn (mut c AtlasClient) copy_pages(collection_name string, page_name string, destination_path string) ! { + // Get page links from metadata + links := c.get_page_links(collection_name, page_name)! + + // Create img subdirectory + mut img_dest := pathlib.get_dir(path: '${destination_path}', create: true)! + + // Copy only image links + for link in links { + if link.file_type != .page { + continue + } + if link.status == .external { + continue + } + // Get image path and copy + img_path := c.get_page_path(link.target_collection_name, link.target_item_name)! + mut src := pathlib.get_file(path: img_path)! + src.copy(dest: '${img_dest.path}/${src.name_fix_keepext()}')! + console.print_debug(' ********. Copied page: ${src.path} to ${img_dest.path}/${src.name_fix_keepext()}') + } +} + + pub fn (mut c AtlasClient) copy_images(collection_name string, page_name string, destination_path string) ! { // Get page links from metadata links := c.get_page_links(collection_name, page_name)! diff --git a/lib/data/atlas/client/client_links.v b/lib/data/atlas/client/client_links.v new file mode 100644 index 00000000..520acdc8 --- /dev/null +++ b/lib/data/atlas/client/client_links.v @@ -0,0 +1,119 @@ +module client + +import incubaid.herolib.core.pathlib +import incubaid.herolib.core.texttools +import incubaid.herolib.ui.console +import os +import json +import incubaid.herolib.core.redisclient + +// get_page_links returns all links found in a page and pages linked to it (recursive) +// This includes transitive links through page-to-page references +// External links, files, and images do not recurse further +pub fn (mut c AtlasClient) get_page_links(collection_name string, page_name string) ![]LinkMetadata { + mut visited := map[string]bool{} + mut all_links := []LinkMetadata{} + c.collect_page_links_recursive(collection_name, page_name, mut visited, mut all_links)! + return all_links +} + + +// collect_page_links_recursive is the internal recursive implementation +// It traverses all linked pages and collects all links found +// +// Thread safety: Each call to get_page_links gets its own visited map +// Circular references are prevented by tracking visited pages +// +// Link types behavior: +// - .page links: Recursively traverse to get links from the target page +// - .file and .image links: Included in results but not recursively expanded +// - .external links: Included in results but not recursively expanded +fn (mut c AtlasClient) collect_page_links_recursive(collection_name string, page_name string, mut visited map[string]bool, mut all_links []LinkMetadata) ! { + // Create unique key for cycle detection + page_key := '${collection_name}:${page_name}' + + // Prevent infinite loops on circular page references + // Example: Page A → Page B → Page A + if page_key in visited { + return + } + visited[page_key] = true + + // Get collection metadata + metadata := c.get_collection_metadata(collection_name)! + fixed_page_name := texttools.name_fix_no_ext(page_name) + + // Find the page in metadata + if fixed_page_name !in metadata.pages { + return error('page_not_found: Page "${page_name}" not found in collection metadata, for collection: "${collection_name}"') + } + + page_meta := metadata.pages[fixed_page_name] + + // Add all direct links from this page to the result + // This includes: pages, files, images, and external links + all_links << page_meta.links + + // Recursively traverse only page-to-page links + for link in page_meta.links { + // Only recursively process links to other pages within the atlas + // Skip external links (http, https, mailto, etc.) + // Skip file and image links (these don't have "contained" links) + if link.file_type != .page || link.status == .external { + continue + } + + // Recursively collect links from the target page + c.collect_page_links_recursive(link.target_collection_name, link.target_item_name, mut visited, mut all_links) or { + // If we encounter an error (e.g., target page doesn't exist in metadata), + // we continue processing other links rather than failing completely + // This provides graceful degradation for broken link references + continue + } + } +} + +// get_image_links returns all image links found in a page and related pages (recursive) +// This is a convenience function that filters get_page_links to only image links +pub fn (mut c AtlasClient) get_image_links(collection_name string, page_name string) ![]LinkMetadata { + all_links := c.get_page_links(collection_name, page_name)! + mut image_links := []LinkMetadata{} + + for link in all_links { + if link.file_type == .image { + image_links << link + } + } + + return image_links +} + +// get_file_links returns all file links (non-image) found in a page and related pages (recursive) +// This is a convenience function that filters get_page_links to only file links +pub fn (mut c AtlasClient) get_file_links(collection_name string, page_name string) ![]LinkMetadata { + all_links := c.get_page_links(collection_name, page_name)! + mut file_links := []LinkMetadata{} + + for link in all_links { + if link.file_type == .file { + file_links << link + } + } + + return file_links +} + +// get_page_link_targets returns all page-to-page link targets found in a page and related pages +// This is a convenience function that filters get_page_links to only page links +pub fn (mut c AtlasClient) get_page_link_targets(collection_name string, page_name string) ![]LinkMetadata { + all_links := c.get_page_links(collection_name, page_name)! + mut page_links := []LinkMetadata{} + + for link in all_links { + if link.file_type == .page && link.status != .external { + page_links << link + } + } + + return page_links +} \ No newline at end of file diff --git a/lib/data/atlas/export.v b/lib/data/atlas/export.v index ac21479d..b10eca31 100644 --- a/lib/data/atlas/export.v +++ b/lib/data/atlas/export.v @@ -7,7 +7,7 @@ import json @[params] pub struct ExportArgs { pub mut: - destination string @[requireds] + destination string @[required] reset bool = true include bool = true redis bool = true @@ -90,6 +90,44 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! { c.collect_cross_collection_references(mut page, mut cross_collection_pages, mut cross_collection_files, mut processed_cross_pages)! + // println('------- ${c.name} ${page.key()}') + // if page.key() == 'geoaware:solution' && c.name == 'mycelium_nodes_tiers' { + // println(cross_collection_pages) + // println(cross_collection_files) + // // println(processed_cross_pages) + // $dbg; + // } + + // copy the pages to the right exported path + for _, mut ref_page in cross_collection_pages { + mut src_file := ref_page.path()! + mut subdir_path := pathlib.get_dir( + path: '${col_dir.path}' + create: true + )! + mut dest_path := '${subdir_path.path}/${ref_page.name}.md' + src_file.copy(dest: dest_path)! + // println(dest_path) + // $dbg; + } + // copy the files to the right exported path + for _, mut ref_file in cross_collection_files { + mut src_file2 := ref_file.path()! + + // Determine subdirectory based on file type + mut subdir := if ref_file.is_image() { 'img' } else { 'files' } + + // Ensure subdirectory exists + mut subdir_path := pathlib.get_dir( + path: '${col_dir.path}/${subdir}' + create: true + )! + + mut dest_path := '${subdir_path.path}/${ref_file.name}' + mut dest_file2 := pathlib.get_file(path: dest_path, create: true)! + src_file2.copy(dest: dest_file2.path)! + } + processed_local_pages[page.name] = true // Redis operations... @@ -117,65 +155,6 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! { mut dest_file := pathlib.get_file(path: dest_path, create: true)! src_file.copy(dest: dest_file.path)! } - - // Second pass: copy all collected cross-collection pages and process their links recursively - // Keep iterating until no new cross-collection references are found - for { - mut found_new_references := false - - // Process all cross-collection pages we haven't processed yet - for page_key, mut ref_page in cross_collection_pages { - if page_key in processed_cross_pages { - continue // Already processed this page's links - } - - // Mark as processed to avoid infinite loops - processed_cross_pages[page_key] = true - found_new_references = true - - // Get the referenced page content with includes processed - ref_content := ref_page.content_with_fixed_links( - include: args.include - cross_collection: true - export_mode: true - )! - - // Write the referenced page to this collection's directory - mut dest_file := pathlib.get_file( - path: '${col_dir.path}/${ref_page.name}.md' - create: true - )! - dest_file.write(ref_content)! - - // CRITICAL: Recursively process links in this cross-collection page - // This ensures we get pages/files/images referenced by ref_page - c.collect_cross_collection_references(mut ref_page, mut cross_collection_pages, mut - cross_collection_files, mut processed_cross_pages)! - } - - // If we didn't find any new references, we're done with the recursive pass - if !found_new_references { - break - } - } - - // Third pass: copy ALL collected cross-collection referenced files/images - for _, mut ref_file in cross_collection_files { - mut src_file := ref_file.path()! - - // Determine subdirectory based on file type - mut subdir := if ref_file.is_image() { 'img' } else { 'files' } - - // Ensure subdirectory exists - mut subdir_path := pathlib.get_dir( - path: '${col_dir.path}/${subdir}' - create: true - )! - - mut dest_path := '${subdir_path.path}/${ref_file.name}' - mut dest_file := pathlib.get_file(path: dest_path, create: true)! - src_file.copy(dest: dest_file.path)! - } } // Helper function to recursively collect cross-collection references @@ -184,6 +163,17 @@ fn (mut c Collection) collect_cross_collection_references(mut page Page, mut all_cross_pages map[string]&Page, mut all_cross_files map[string]&File, mut processed_pages map[string]bool) ! { + page_key := page.key() + + // If we've already processed this page, skip it (prevents infinite loops with cycles) + if page_key in processed_pages { + return + } + + // Mark this page as processed BEFORE recursing (prevents infinite loops with circular references) + processed_pages[page_key] = true + + // Process all links in the current page // Use cached links from validation (before transformation) to preserve collection info for mut link in page.links { if link.status != .found { @@ -192,15 +182,19 @@ fn (mut c Collection) collect_cross_collection_references(mut page Page, is_local := link.target_collection_name == c.name - // Collect cross-collection page references + // Collect cross-collection page references and recursively process them if link.file_type == .page && !is_local { - page_key := '${link.target_collection_name}:${link.target_item_name}' + page_ref := '${link.target_collection_name}:${link.target_item_name}' // Only add if not already collected - if page_key !in all_cross_pages { + if page_ref !in all_cross_pages { mut target_page := link.target_page()! - all_cross_pages[page_key] = target_page - // Don't mark as processed yet - we'll do that when we actually process its links + all_cross_pages[page_ref] = target_page + + // Recursively process the target page's links to find more cross-collection references + // This ensures we collect ALL transitive cross-collection page and file references + c.collect_cross_collection_references(mut target_page, mut all_cross_pages, mut + all_cross_files, mut processed_pages)! } } diff --git a/lib/data/atlas/instruction.md b/lib/data/atlas/instruction.md deleted file mode 100644 index f0ae7f35..00000000 --- a/lib/data/atlas/instruction.md +++ /dev/null @@ -1,15 +0,0 @@ -in atlas/ - -check format of groups -see content/groups - -now the groups end with .group - -check how the include works, so we can include another group in the group as defined, only works in same folder - -in the scan function in atlas, now make scan_groups function, find groups, only do this for collection as named groups -do not add collection groups to atlas, this is a system collection - -make the groups and add them to atlas - -give clear instructions for coding agent how to write the code diff --git a/lib/data/atlas/play.v b/lib/data/atlas/play.v index cd84244a..034e4fcd 100644 --- a/lib/data/atlas/play.v +++ b/lib/data/atlas/play.v @@ -3,6 +3,7 @@ module atlas import incubaid.herolib.core.playbook { PlayBook } import incubaid.herolib.develop.gittools import incubaid.herolib.ui.console +import os // Play function to process HeroScript actions for Atlas pub fn play(mut plbook PlayBook) ! { @@ -66,7 +67,7 @@ pub fn play(mut plbook PlayBook) ! { for mut action in export_actions { mut p := action.params name = p.get_default('name', 'main')! - destination := p.get_default('destination', '/tmp/atlas_export')! + destination := p.get_default('destination', '${os.home_dir()}/hero/var/atlas_export')! reset := p.get_default_true('reset') include := p.get_default_true('include') redis := p.get_default_true('redis') diff --git a/lib/data/atlas/process.md b/lib/data/atlas/process.md deleted file mode 100644 index 1730a12c..00000000 --- a/lib/data/atlas/process.md +++ /dev/null @@ -1,4 +0,0 @@ - - -- first find all pages -- then for each page find all links \ No newline at end of file diff --git a/lib/data/atlas/readme.md b/lib/data/atlas/readme.md index dbb27a6b..fb9a8817 100644 --- a/lib/data/atlas/readme.md +++ b/lib/data/atlas/readme.md @@ -33,7 +33,7 @@ put in .hero file and execute with hero or but shebang line on top of .hero scri !!atlas.scan git_url:"https://git.ourworld.tf/tfgrid/docs_tfgrid4/src/branch/main/collections/tests" -!!atlas.export destination: '/tmp/atlas_export' +!!atlas.export ``` diff --git a/lib/web/docusaurus/dsite.v b/lib/web/docusaurus/dsite.v index 80f48226..fb8dcf4f 100644 --- a/lib/web/docusaurus/dsite.v +++ b/lib/web/docusaurus/dsite.v @@ -71,9 +71,9 @@ pub struct DevArgs { pub mut: host string = 'localhost' port int = 3000 - open bool = true // whether to open the browser automatically - watch_changes bool = false // whether to watch for changes in docs and rebuild automatically - skip_generate bool = false // whether to skip generation (useful when docs are pre-generated, e.g., from atlas) + open bool = true // whether to open the browser automatically + watch_changes bool // whether to watch for changes in docs and rebuild automatically + skip_generate bool // whether to skip generation (useful when docs are pre-generated, e.g., from atlas) } pub fn (mut s DocSite) open(args DevArgs) ! { diff --git a/lib/web/docusaurus/dsite_generate_docs.v b/lib/web/docusaurus/dsite_generate_docs.v index 3fb548d7..b5f20d25 100644 --- a/lib/web/docusaurus/dsite_generate_docs.v +++ b/lib/web/docusaurus/dsite_generate_docs.v @@ -142,6 +142,10 @@ fn (mut generator SiteGenerator) page_generate(args_ Page) ! { pagefile.write(c)! + generator.client.copy_pages(collection_name, page_name, pagefile.path_dir()) or { + generator.error("Couldn't copy pages for page:'${page_name}' in collection:'${collection_name}'\nERROR:${err}")! + return + } generator.client.copy_images(collection_name, page_name, pagefile.path_dir()) or { generator.error("Couldn't copy images for page:'${page_name}' in collection:'${collection_name}'\nERROR:${err}")! return diff --git a/lib/web/docusaurus/interface_atlas_client.v b/lib/web/docusaurus/interface_atlas_client.v deleted file mode 100644 index c687d884..00000000 --- a/lib/web/docusaurus/interface_atlas_client.v +++ /dev/null @@ -1,30 +0,0 @@ -module docusaurus - -pub interface IDocClient { -mut: - // Path methods - get absolute paths to resources - get_page_path(collection_name string, page_name string) !string - get_file_path(collection_name string, file_name string) !string - get_image_path(collection_name string, image_name string) !string - - // Existence checks - verify if resources exist - page_exists(collection_name string, page_name string) bool - file_exists(collection_name string, file_name string) bool - image_exists(collection_name string, image_name string) bool - - // Content retrieval - get_page_content(collection_name string, page_name string) !string - - // Listing methods - enumerate resources - list_collections() ![]string - list_pages(collection_name string) ![]string - list_files(collection_name string) ![]string - list_images(collection_name string) ![]string - list_pages_map() !map[string][]string - list_markdown() !string - - // Image operations - // get_page_paths(collection_name string, page_name string) !(string, []string) - copy_images(collection_name string, page_name string, destination_path string) ! - copy_files(collection_name string, page_name string, destination_path string) ! -} diff --git a/lib/web/docusaurus/play.v b/lib/web/docusaurus/play.v index 37609f9a..14b6f0f1 100644 --- a/lib/web/docusaurus/play.v +++ b/lib/web/docusaurus/play.v @@ -1,6 +1,7 @@ module docusaurus import incubaid.herolib.core.playbook { PlayBook } +import os pub fn play(mut plbook PlayBook) ! { if !plbook.exists(filter: 'docusaurus.') { @@ -17,7 +18,7 @@ pub fn play(mut plbook PlayBook) ! { reset: param_define.get_default_false('reset') template_update: param_define.get_default_false('template_update') install: param_define.get_default_false('install') - atlas_dir: param_define.get_default('atlas_dir', '/tmp/atlas_export')! + atlas_dir: param_define.get_default('atlas_dir', '${os.home_dir()}/hero/var/atlas_export')! use_atlas: param_define.get_default_false('use_atlas') )! diff --git a/lib/web/site/ai_instructions.md b/lib/web/site/ai_instructions.md deleted file mode 100644 index 8db0c377..00000000 --- a/lib/web/site/ai_instructions.md +++ /dev/null @@ -1,536 +0,0 @@ -# AI Instructions for Site Module HeroScript - -This document provides comprehensive instructions for AI agents working with the Site module's HeroScript format. - -## HeroScript Format Overview - -HeroScript is a declarative configuration language with the following characteristics: - -### Basic Syntax - -```heroscript -!!actor.action - param1: "value1" - param2: "value2" - multiline_param: " - This is a multiline value. - It can span multiple lines. - " - arg1 arg2 // Arguments without keys -``` - -**Key Rules:** -1. Actions start with `!!` followed by `actor.action` format -2. Parameters are indented and use `key: "value"` or `key: value` format -3. Values with spaces must be quoted -4. Multiline values are supported with quotes -5. Arguments without keys are space-separated -6. Comments start with `//` - -## Site Module Actions - -### 1. Site Configuration (`!!site.config`) - -**Purpose:** Define the main site configuration including title, description, and metadata. - -**Required Parameters:** -- `name`: Site identifier (will be normalized to snake_case) - -**Optional Parameters:** -- `title`: Site title (default: "Documentation Site") -- `description`: Site description -- `tagline`: Site tagline -- `favicon`: Path to favicon (default: "img/favicon.png") -- `image`: Default site image (default: "img/tf_graph.png") -- `copyright`: Copyright text -- `url`: Main site URL -- `base_url`: Base URL path (default: "/") -- `url_home`: Home page path - -**Example:** -```heroscript -!!site.config - name: "my_documentation" - title: "My Documentation Site" - description: "Comprehensive technical documentation" - tagline: "Learn everything you need" - url: "https://docs.example.com" - base_url: "/" -``` - -**AI Guidelines:** -- Always include `name` parameter -- Use descriptive titles and descriptions -- Ensure URLs are properly formatted with protocol - -### 2. Metadata Configuration (`!!site.config_meta`) - -**Purpose:** Override specific metadata for SEO purposes. - -**Optional Parameters:** -- `title`: SEO-specific title (overrides site.config title for meta tags) -- `image`: SEO-specific image (overrides site.config image for og:image) -- `description`: SEO-specific description - -**Example:** -```heroscript -!!site.config_meta - title: "My Docs - Complete Guide" - image: "img/social-preview.png" - description: "The ultimate guide to using our platform" -``` - -**AI Guidelines:** -- Use only when SEO metadata needs to differ from main config -- Keep titles concise for social media sharing -- Use high-quality images for social previews - -### 3. Navigation Bar (`!!site.navbar` or `!!site.menu`) - -**Purpose:** Configure the main navigation bar. - -**Optional Parameters:** -- `title`: Navigation title (defaults to site.config title) -- `logo_alt`: Logo alt text -- `logo_src`: Logo image path -- `logo_src_dark`: Dark mode logo path - -**Example:** -```heroscript -!!site.navbar - title: "My Site" - logo_alt: "My Site Logo" - logo_src: "img/logo.svg" - logo_src_dark: "img/logo-dark.svg" -``` - -**AI Guidelines:** -- Use `!!site.navbar` for modern syntax (preferred) -- `!!site.menu` is supported for backward compatibility -- Provide both light and dark logos when possible - -### 4. Navigation Items (`!!site.navbar_item` or `!!site.menu_item`) - -**Purpose:** Add items to the navigation bar. - -**Required Parameters (one of):** -- `to`: Internal link path -- `href`: External URL - -**Optional Parameters:** -- `label`: Display text (required in practice) -- `position`: "left" or "right" (default: "right") - -**Example:** -```heroscript -!!site.navbar_item - label: "Documentation" - to: "docs/intro" - position: "left" - -!!site.navbar_item - label: "GitHub" - href: "https://github.com/myorg/repo" - position: "right" -``` - -**AI Guidelines:** -- Use `to` for internal navigation -- Use `href` for external links -- Position important items on the left, secondary items on the right - -### 5. Footer Configuration (`!!site.footer`) - -**Purpose:** Configure footer styling. - -**Optional Parameters:** -- `style`: "dark" or "light" (default: "dark") - -**Example:** -```heroscript -!!site.footer - style: "dark" -``` - -### 6. Footer Items (`!!site.footer_item`) - -**Purpose:** Add links to the footer, grouped by title. - -**Required Parameters:** -- `title`: Group title (items with same title are grouped together) -- `label`: Link text - -**Required Parameters (one of):** -- `to`: Internal link path -- `href`: External URL - -**Example:** -```heroscript -!!site.footer_item - title: "Docs" - label: "Introduction" - to: "intro" - -!!site.footer_item - title: "Docs" - label: "API Reference" - to: "api" - -!!site.footer_item - title: "Community" - label: "Discord" - href: "https://discord.gg/example" -``` - -**AI Guidelines:** -- Group related links under the same title -- Use consistent title names across related items -- Provide both internal and external links as appropriate - -### 7. Page Categories (`!!site.page_category`) - -**Purpose:** Create a section/category to organize pages. - -**Required Parameters:** -- `name`: Category identifier (snake_case) - -**Optional Parameters:** -- `label`: Display name (auto-generated from name if not provided) -- `position`: Manual sort order (auto-incremented if not specified) -- `path`: URL path segment (defaults to normalized label) - -**Example:** -```heroscript -!!site.page_category - name: "getting_started" - label: "Getting Started" - position: 100 - -!!site.page_category - name: "advanced_topics" - label: "Advanced Topics" -``` - -**AI Guidelines:** -- Use descriptive snake_case names -- Let label be auto-generated when possible (name_fix converts to Title Case) -- Categories persist for all subsequent pages until a new category is declared -- Position values should leave gaps (100, 200, 300) for future insertions - -### 8. Pages (`!!site.page`) - -**Purpose:** Define individual pages in the site. - -**Required Parameters:** -- `src`: Source reference as `collection:page_name` (required for first page in a collection) - -**Optional Parameters:** -- `name`: Page identifier (extracted from src if not provided) -- `title`: Page title (extracted from markdown if not provided) -- `description`: Page description for metadata -- `slug`: Custom URL slug -- `position`: Manual sort order (auto-incremented if not specified) -- `draft`: Mark as draft (default: false) -- `hide_title`: Hide title in rendering (default: false) -- `path`: Custom path (defaults to current category name) -- `category`: Override current category -- `title_nr`: Title numbering level - -**Example:** -```heroscript -!!site.page src: "docs:introduction" - description: "Introduction to the platform" - slug: "/" - -!!site.page src: "quickstart" - description: "Get started in 5 minutes" - -!!site.page src: "installation" - title: "Installation Guide" - description: "How to install and configure" - position: 10 -``` - -**AI Guidelines:** -- **Collection Persistence:** Specify collection once (e.g., `docs:introduction`), then subsequent pages only need page name (e.g., `quickstart`) -- **Category Persistence:** Pages belong to the most recently declared category -- **Title Extraction:** Prefer extracting titles from markdown files -- **Position Management:** Use automatic positioning unless specific order is required -- **Description Required:** Always provide descriptions for SEO -- **Slug Usage:** Use slug for special pages like homepage (`slug: "/"`) - -### 9. Import External Content (`!!site.import`) - -**Purpose:** Import content from external sources. - -**Optional Parameters:** -- `name`: Import identifier -- `url`: Git URL or HTTP URL -- `path`: Local file system path -- `dest`: Destination path in site -- `replace`: Comma-separated key:value pairs for variable replacement -- `visible`: Whether imported content is visible (default: true) - -**Example:** -```heroscript -!!site.import - url: "https://github.com/example/docs" - dest: "external" - replace: "VERSION:1.0.0,PROJECT:MyProject" - visible: true -``` - -**AI Guidelines:** -- Use for shared documentation across multiple sites -- Replace variables using `${VARIABLE}` syntax in source content -- Set `visible: false` for imported templates or partials - -### 10. Publish Destinations (`!!site.publish` and `!!site.publish_dev`) - -**Purpose:** Define where to publish the built site. - -**Optional Parameters:** -- `path`: File system path or URL -- `ssh_name`: SSH connection name for remote deployment - -**Example:** -```heroscript -!!site.publish - path: "/var/www/html/docs" - ssh_name: "production_server" - -!!site.publish_dev - path: "/tmp/docs-preview" -``` - -**AI Guidelines:** -- Use `!!site.publish` for production deployments -- Use `!!site.publish_dev` for development/preview deployments -- Can specify multiple destinations - -## File Organization Best Practices - -### Naming Convention - -Use numeric prefixes to control execution order: - -``` -0_config.heroscript # Site configuration -1_navigation.heroscript # Menu and footer -2_intro.heroscript # Introduction pages -3_guides.heroscript # User guides -4_reference.heroscript # API reference -``` - -**AI Guidelines:** -- Always use numeric prefixes (0_, 1_, 2_, etc.) -- Leave gaps in numbering (0, 10, 20) for future insertions -- Group related configurations in the same file -- Process order matters: config → navigation → pages - -### Execution Order Rules - -1. **Configuration First:** `!!site.config` must be processed before other actions -2. **Categories Before Pages:** Declare `!!site.page_category` before pages in that category -3. **Collection Persistence:** First page in a collection must specify `collection:page_name` -4. **Category Persistence:** Pages inherit the most recent category declaration - -## Common Patterns - -### Pattern 1: Simple Documentation Site - -```heroscript -!!site.config - name: "simple_docs" - title: "Simple Documentation" - -!!site.navbar - title: "Simple Docs" - -!!site.page src: "docs:index" - description: "Welcome page" - slug: "/" - -!!site.page src: "getting-started" - description: "Getting started guide" - -!!site.page src: "api" - description: "API reference" -``` - -### Pattern 2: Multi-Section Documentation - -```heroscript -!!site.config - name: "multi_section_docs" - title: "Complete Documentation" - -!!site.page_category - name: "introduction" - label: "Introduction" - -!!site.page src: "docs:welcome" - description: "Welcome to our documentation" - -!!site.page src: "overview" - description: "Platform overview" - -!!site.page_category - name: "tutorials" - label: "Tutorials" - -!!site.page src: "tutorial_basics" - description: "Basic tutorial" - -!!site.page src: "tutorial_advanced" - description: "Advanced tutorial" -``` - -### Pattern 3: Complex Site with External Links - -```heroscript -!!site.config - name: "complex_site" - title: "Complex Documentation Site" - url: "https://docs.example.com" - -!!site.navbar - title: "My Platform" - logo_src: "img/logo.svg" - -!!site.navbar_item - label: "Docs" - to: "docs/intro" - position: "left" - -!!site.navbar_item - label: "API" - to: "api" - position: "left" - -!!site.navbar_item - label: "GitHub" - href: "https://github.com/example/repo" - position: "right" - -!!site.footer - style: "dark" - -!!site.footer_item - title: "Documentation" - label: "Getting Started" - to: "docs/intro" - -!!site.footer_item - title: "Community" - label: "Discord" - href: "https://discord.gg/example" - -!!site.page_category - name: "getting_started" - -!!site.page src: "docs:introduction" - description: "Introduction to the platform" - slug: "/" - -!!site.page src: "installation" - description: "Installation guide" -``` - -## Error Prevention - -### Common Mistakes to Avoid - -1. **Missing Collection on First Page:** - ```heroscript - # WRONG - no collection specified - !!site.page src: "introduction" - - # CORRECT - !!site.page src: "docs:introduction" - ``` - -2. **Category Without Name:** - ```heroscript - # WRONG - missing name - !!site.page_category - label: "Getting Started" - - # CORRECT - !!site.page_category - name: "getting_started" - label: "Getting Started" - ``` - -3. **Missing Description:** - ```heroscript - # WRONG - no description - !!site.page src: "docs:intro" - - # CORRECT - !!site.page src: "docs:intro" - description: "Introduction to the platform" - ``` - -4. **Incorrect File Ordering:** - ``` - # WRONG - pages before config - pages.heroscript - config.heroscript - - # CORRECT - config first - 0_config.heroscript - 1_pages.heroscript - ``` - -## Validation Checklist - -When generating HeroScript for the Site module, verify: - -- [ ] `!!site.config` includes `name` parameter -- [ ] All pages have `description` parameter -- [ ] First page in each collection specifies `collection:page_name` -- [ ] Categories are declared before their pages -- [ ] Files use numeric prefixes for ordering -- [ ] Navigation items have either `to` or `href` -- [ ] Footer items are grouped by `title` -- [ ] External URLs include protocol (https://) -- [ ] Paths don't have trailing slashes unless intentional -- [ ] Draft pages are marked with `draft: true` - -## Integration with V Code - -When working with the Site module in V code: - -```v -import incubaid.herolib.web.site -import incubaid.herolib.core.playbook - -// Process HeroScript files -mut plbook := playbook.new(path: '/path/to/heroscripts')! -site.play(mut plbook)! - -// Access configured site -mut mysite := site.get(name: 'my_site')! - -// Iterate through pages -for page in mysite.pages { - println('Page: ${page.name} - ${page.description}') -} - -// Iterate through sections -for section in mysite.sections { - println('Section: ${section.label}') -} -``` - -## Summary - -The Site module's HeroScript format provides a declarative way to configure websites with: -- Clear separation of concerns (config, navigation, content) -- Automatic ordering and organization -- Collection and category persistence for reduced repetition -- Flexible metadata and SEO configuration -- Support for both internal and external content - -Always follow the execution order rules, use numeric file prefixes, and provide complete metadata for best results. \ No newline at end of file diff --git a/lib/web/site/model_nav.v b/lib/web/site/model_nav.v new file mode 100644 index 00000000..066080b7 --- /dev/null +++ b/lib/web/site/model_nav.v @@ -0,0 +1,145 @@ +module site + +import json + +// Top-level config +pub struct NavConfig { +pub mut: + mySidebar []NavItem + // myTopbar []NavItem //not used yet + // myFooter []NavItem //not used yet +} + +// -------- Variant Type -------- +pub type NavItem = NavDoc | NavCat | NavLink + +// --------- DOC ITEM ---------- +pub struct NavDoc { +pub: + id string //is the page id + label string +} + +// --------- CATEGORY ---------- +pub struct NavCat { +pub mut: + label string + collapsible bool + collapsed bool + items []NavItem +} + +// --------- LINK ---------- +pub struct NavLink { +pub: + label string + href string + description string +} + +// -------- JSON SERIALIZATION -------- + +// NavItemJson is used for JSON export with type discrimination +pub struct NavItemJson { +pub mut: + type_field string @[json: 'type'] + // For doc + id string @[omitempty] + label string @[omitempty] + // For link + href string @[omitempty] + description string @[omitempty] + // For category + collapsible bool + collapsed bool + items []NavItemJson @[omitempty] +} + +// Convert a single NavItem to JSON-serializable format +fn nav_item_to_json(item NavItem) !NavItemJson { + return match item { + NavDoc { + NavItemJson{ + type_field: 'doc' + id: item.id + label: item.label + collapsible: false + collapsed: false + } + } + NavLink { + NavItemJson{ + type_field: 'link' + label: item.label + href: item.href + description: item.description + collapsible: false + collapsed: false + } + } + NavCat { + mut json_items := []NavItemJson{} + for sub_item in item.items { + json_items << nav_item_to_json(sub_item)! + } + NavItemJson{ + type_field: 'category' + label: item.label + collapsible: item.collapsible + collapsed: item.collapsed + items: json_items + } + } + } +} + +// Convert entire NavConfig sidebar to JSON-serializable array +fn (nc NavConfig) sidebar_to_json() ![]NavItemJson { + mut result := []NavItemJson{} + for item in nc.mySidebar { + result << nav_item_to_json(item)! + } + return result +} + + + +// // Convert entire NavConfig topbar to JSON-serializable array +// fn (nc NavConfig) topbar_to_json() ![]NavItemJson { +// mut result := []NavItemJson{} +// for item in nc.myTopbar { +// result << nav_item_to_json(item)! +// } +// return result +// } + +// // Convert entire NavConfig footer to JSON-serializable array +// fn (nc NavConfig) footer_to_json() ![]NavItemJson { +// mut result := []NavItemJson{} +// for item in nc.myFooter { +// result << nav_item_to_json(item)! +// } +// return result +// } + +port topbar as formatted JSON string +// pub fn (nc NavConfig) jsondump_topbar() !string { +// items := nc.topbar_to_json()! +// return json.encode_pretty(items) +// } + +// // Export footer as formatted JSON string +// pub fn (nc NavConfig) jsondump_footer() !string { +// items := nc.footer_to_json()! +// return json.encode_pretty(items) +// } + +// // Export all navigation as object with sidebar, topbar, footer +// pub fn (nc NavConfig) jsondump_all() !string { +// all_nav := map[string][]NavItemJson{ +// 'sidebar': nc.sidebar_to_json()! +// 'topbar': nc.topbar_to_json()! +// 'footer': nc.footer_to_json()! +// } +// return json.encode_pretty(all_nav) +// } diff --git a/lib/web/site/model_page.v b/lib/web/site/model_page.v index 30bfeaed..d82729ab 100644 --- a/lib/web/site/model_page.v +++ b/lib/web/site/model_page.v @@ -2,15 +2,10 @@ module site pub struct Page { pub mut: - name string + id string //unique identifier, by default as "collection:page_name", we can overrule this from play instructions if needed title string description string draft bool - position int hide_title bool src string @[required] // always in format collection:page_name, can use the default collection if no : specified - path string @[required] // is without the page name, so just the path to the folder where the page is in - section_name string - title_nr int - slug string } diff --git a/lib/web/site/model_site.v b/lib/web/site/model_site.v new file mode 100644 index 00000000..81665a06 --- /dev/null +++ b/lib/web/site/model_site.v @@ -0,0 +1,9 @@ +module site + +@[heap] +pub struct Site { +pub mut: + pages map[string]Page // key: is the id of the page + nav NavConfig //navigation of the site + siteconfig SiteConfig +} diff --git a/lib/web/site/model_site_section.v b/lib/web/site/model_site_section.v deleted file mode 100644 index df491fa0..00000000 --- a/lib/web/site/model_site_section.v +++ /dev/null @@ -1,18 +0,0 @@ -module site - -@[heap] -pub struct Site { -pub mut: - pages []Page - sections []Section - siteconfig SiteConfig -} - -pub struct Section { -pub mut: - name string - position int - path string - label string - description string -}