MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

...

Browse Source
This commit is contained in:
despiegk
2025-11-28 09:01:58 +01:00
parent 60e2230448
commit 0414ea85df
18 changed files with 679 additions and 696 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
examples/data/atlas/debug_recursive/debug_atlas.vsh Normal file
View File

@@ -0,0 +1,308 @@
#!/usr/bin/env -S vrun
import incubaid.herolib.data.atlas
import incubaid.herolib.ui.console
import os
fn main() {
println('=== ATLAS DEBUG SCRIPT ===\n')
// Create and scan atlas
mut a := atlas.new(name: 'main')!
// Scan the collections
println('Scanning collections...\n')
a.scan(
path: '/Users/despiegk/code/git.ourworld.tf/geomind/docs_geomind/collections/mycelium_nodes_tiers'
)!
a.scan(
path: '/Users/despiegk/code/git.ourworld.tf/geomind/docs_geomind/collections/geomind_compare'
)!
a.scan(path: '/Users/despiegk/code/git.ourworld.tf/geomind/docs_geomind/collections/geoaware')!
a.scan(
path: '/Users/despiegk/code/git.ourworld.tf/tfgrid/docs_tfgrid4/collections/mycelium_economics'
)!
a.scan(
path: '/Users/despiegk/code/git.ourworld.tf/tfgrid/docs_tfgrid4/collections/mycelium_concepts'
)!
a.scan(
path: '/Users/despiegk/code/git.ourworld.tf/tfgrid/docs_tfgrid4/collections/mycelium_cloud_tech'
)!
// Initialize atlas (post-scanning validation)
a.init_post()!
// Print all pages per collection
println('\n=== COLLECTIONS & PAGES ===\n')
for col_name, col in a.collections {
println('Collection: ${col_name}')
println(' Pages (${col.pages.len}):')
if col.pages.len > 0 {
for page_name, _ in col.pages {
println(' - ${page_name}')
}
} else {
println(' (empty)')
}
println(' Files/Images (${col.files.len}):')
if col.files.len > 0 {
for file_name, _ in col.files {
println(' - ${file_name}')
}
} else {
println(' (empty)')
}
}
// Validate links (this will recursively find links across collections)
println('\n=== VALIDATING LINKS (RECURSIVE) ===\n')
a.validate_links()!
println('✓ Link validation complete\n')
// Check for broken links
println('\n=== BROKEN LINKS ===\n')
mut total_errors := 0
for col_name, col in a.collections {
if col.has_errors() {
println('Collection: ${col_name} (${col.errors.len} errors)')
for err in col.errors {
println(' [${err.category_str()}] Page: ${err.page_key}')
println(' Message: ${err.message}')
println('')
total_errors++
}
}
}
if total_errors == 0 {
println('✓ No broken links found!')
} else {
println('\n❌ Total broken link errors: ${total_errors}')
}
// Show discovered links per page (validates recursive discovery)
println('\n\n=== DISCOVERED LINKS (RECURSIVE RESOLUTION) ===\n')
println('Checking for files referenced by cross-collection pages...\n')
mut total_links := 0
for col_name, col in a.collections {
mut col_has_links := false
for page_name, page in col.pages {
if page.links.len > 0 {
if !col_has_links {
println('Collection: ${col_name}')
col_has_links = true
}
println(' Page: ${page_name} (${page.links.len} links)')
for link in page.links {
target_col := if link.target_collection_name != '' {
link.target_collection_name
} else {
col_name
}
println(' → ${target_col}:${link.target_item_name} [${link.file_type}]')
total_links++
}
}
}
}
println('\n✓ Total links discovered: ${total_links}')
// List pages that need investigation
println('\n=== CHECKING SPECIFIC MISSING PAGES ===\n')
missing_pages := [
'compare_electricity',
'internet_basics',
'centralization_risk',
'gdp_negative',
]
// Check in geoaware collection
if 'geoaware' in a.collections {
mut geoaware := a.get_collection('geoaware')!
println('Collection: geoaware')
if geoaware.pages.len > 0 {
println(' All pages in collection:')
for page_name, _ in geoaware.pages {
println(' - ${page_name}')
}
} else {
println(' (No pages found)')
}
println('\n Checking for specific missing pages:')
for page_name in missing_pages {
exists := page_name in geoaware.pages
status := if exists { '✓' } else { '✗' }
println(' ${status} ${page_name}')
}
}
// Check for pages across all collections
println('\n\n=== LOOKING FOR MISSING PAGES ACROSS ALL COLLECTIONS ===\n')
for missing_page in missing_pages {
println('Searching for "${missing_page}":')
mut found := false
for col_name, col in a.collections {
if missing_page in col.pages {
println(' ✓ Found in: ${col_name}')
found = true
}
}
if !found {
println(' ✗ Not found in any collection')
}
}
// Check for the solution page
println('\n\n=== CHECKING FOR "solution" PAGE ===\n')
for col_name in ['mycelium_nodes_tiers', 'geomind_compare', 'geoaware', 'mycelium_economics',
'mycelium_concepts', 'mycelium_cloud_tech'] {
if col_name in a.collections {
mut col := a.get_collection(col_name)!
exists := col.page_exists('solution')!
status := if exists { '✓' } else { '✗' }
println('${status} ${col_name}: "solution" page')
}
}
// Print error summary
println('\n\n=== ERROR SUMMARY BY CATEGORY ===\n')
mut category_counts := map[string]int{}
for _, col in a.collections {
for err in col.errors {
cat_str := err.category_str()
category_counts[cat_str]++
}
}
if category_counts.len == 0 {
println('✓ No errors found!')
} else {
for cat, count in category_counts {
println('${cat}: ${count}')
}
}
// ===== EXPORT AND FILE VERIFICATION TEST =====
println('\n\n=== EXPORT AND FILE VERIFICATION TEST ===\n')
// Create export directory
export_path := '/tmp/atlas_debug_export'
if os.exists(export_path) {
os.rmdir_all(export_path)!
}
os.mkdir_all(export_path)!
println('Exporting to: ${export_path}\n')
a.export(destination: export_path)!
println('✓ Export completed\n')
// Collect all files found during link validation
mut expected_files := map[string]string{} // key: file_name, value: collection_name
mut file_count := 0
for col_name, col in a.collections {
for page_name, page in col.pages {
for link in page.links {
if link.status == .found && (link.file_type == .file || link.file_type == .image) {
file_key := link.target_item_name
expected_files[file_key] = link.target_collection_name
file_count++
}
}
}
}
println('Expected to find ${file_count} file references in links\n')
println('=== VERIFYING FILES IN EXPORT DIRECTORY ===\n')
// Get the first collection name (the primary exported collection)
mut primary_col_name := ''
for col_name, _ in a.collections {
primary_col_name = col_name
break
}
if primary_col_name == '' {
println('❌ No collections found')
} else {
mut verified_count := 0
mut missing_count := 0
mut found_files := map[string]bool{}
// Check both img and files directories
img_dir := '${export_path}/content/${primary_col_name}/img'
files_dir := '${export_path}/content/${primary_col_name}/files'
// Scan img directory
if os.exists(img_dir) {
img_files := os.ls(img_dir) or { []string{} }
for img_file in img_files {
found_files[img_file] = true
}
}
// Scan files directory
if os.exists(files_dir) {
file_list := os.ls(files_dir) or { []string{} }
for file in file_list {
found_files[file] = true
}
}
println('Files/Images found in export directory:')
if found_files.len > 0 {
for file_name, _ in found_files {
println(' ✓ ${file_name}')
if file_name in expected_files {
verified_count++
}
}
} else {
println(' (none found)')
}
println('\n=== FILE VERIFICATION RESULTS ===\n')
println('Expected files from links: ${file_count}')
println('Files found in export: ${found_files.len}')
println('Files verified (present in export): ${verified_count}')
// Check for missing expected files
for expected_file, source_col in expected_files {
if expected_file !in found_files {
missing_count++
println(' ✗ Missing: ${expected_file} (from ${source_col})')
}
}
if missing_count > 0 {
println('\n❌ ${missing_count} expected files are MISSING from export!')
} else if verified_count == file_count && file_count > 0 {
println('\n✓ All expected files are present in export directory!')
} else if file_count == 0 {
println('\n⚠ No file links were found during validation (check if pages have file references)')
}
// Show directory structure
println('\n=== EXPORT DIRECTORY STRUCTURE ===\n')
if os.exists('${export_path}/content/${primary_col_name}') {
println('${export_path}/content/${primary_col_name}/')
content_files := os.ls('${export_path}/content/${primary_col_name}') or { []string{} }
for item in content_files {
full_path := '${export_path}/content/${primary_col_name}/${item}'
if os.is_dir(full_path) {
sub_items := os.ls(full_path) or { []string{} }
println(' ${item}/ (${sub_items.len} items)')
for sub_item in sub_items {
println(' - ${sub_item}')
}
} else {
println(' - ${item}')
}
}
}
}
}

4
lib/data/atlas/client/README.md
View File

@@ -17,8 +17,8 @@ AtlasClient provides methods to:
```v
import incubaid.herolib.web.atlas_client
// Create client
mut client := atlas_client.new(export_dir: '/tmp/atlas_export')!
// Create client, exports will be in $/hero/var/atlas_export by default
mut client := atlas_client.new()!
// List collections
collections := client.list_collections()!

38
lib/data/atlas/client/client.v
View File

@@ -247,20 +247,6 @@ pub fn (mut c AtlasClient) get_collection_metadata(collection_name string) !Coll
return metadata
}
// get_page_links returns the links found in a page by reading the metadata
pub fn (mut c AtlasClient) get_page_links(collection_name string, page_name string) ![]LinkMetadata {
// Get collection metadata
metadata := c.get_collection_metadata(collection_name)!
// Apply name normalization to page name
fixed_page_name := texttools.name_fix_no_ext(page_name)
// Find the page in metadata
if fixed_page_name in metadata.pages {
return metadata.pages[fixed_page_name].links
}
return error('page_not_found: Page "${page_name}" not found in collection metadata, for collection: "${collection_name}"')
}
// get_collection_errors returns the errors for a collection from metadata
pub fn (mut c AtlasClient) get_collection_errors(collection_name string) ![]ErrorMetadata {
metadata := c.get_collection_metadata(collection_name)!
@@ -273,6 +259,30 @@ pub fn (mut c AtlasClient) has_errors(collection_name string) bool {
return errors.len > 0
}
pub fn (mut c AtlasClient) copy_pages(collection_name string, page_name string, destination_path string) ! {
// Get page links from metadata
links := c.get_page_links(collection_name, page_name)!
// Create img subdirectory
mut img_dest := pathlib.get_dir(path: '${destination_path}', create: true)!
// Copy only image links
for link in links {
if link.file_type != .page {
continue
}
if link.status == .external {
continue
}
// Get image path and copy
img_path := c.get_page_path(link.target_collection_name, link.target_item_name)!
mut src := pathlib.get_file(path: img_path)!
src.copy(dest: '${img_dest.path}/${src.name_fix_keepext()}')!
console.print_debug(' ********. Copied page: ${src.path} to ${img_dest.path}/${src.name_fix_keepext()}')
}
}
pub fn (mut c AtlasClient) copy_images(collection_name string, page_name string, destination_path string) ! {
// Get page links from metadata
links := c.get_page_links(collection_name, page_name)!

119
lib/data/atlas/client/client_links.v Normal file
View File

@@ -0,0 +1,119 @@
module client
import incubaid.herolib.core.pathlib
import incubaid.herolib.core.texttools
import incubaid.herolib.ui.console
import os
import json
import incubaid.herolib.core.redisclient
// get_page_links returns all links found in a page and pages linked to it (recursive)
// This includes transitive links through page-to-page references
// External links, files, and images do not recurse further
pub fn (mut c AtlasClient) get_page_links(collection_name string, page_name string) ![]LinkMetadata {
mut visited := map[string]bool{}
mut all_links := []LinkMetadata{}
c.collect_page_links_recursive(collection_name, page_name, mut visited, mut all_links)!
return all_links
}
// collect_page_links_recursive is the internal recursive implementation
// It traverses all linked pages and collects all links found
//
// Thread safety: Each call to get_page_links gets its own visited map
// Circular references are prevented by tracking visited pages
//
// Link types behavior:
// - .page links: Recursively traverse to get links from the target page
// - .file and .image links: Included in results but not recursively expanded
// - .external links: Included in results but not recursively expanded
fn (mut c AtlasClient) collect_page_links_recursive(collection_name string, page_name string, mut visited map[string]bool, mut all_links []LinkMetadata) ! {
// Create unique key for cycle detection
page_key := '${collection_name}:${page_name}'
// Prevent infinite loops on circular page references
// Example: Page A → Page B → Page A
if page_key in visited {
return
}
visited[page_key] = true
// Get collection metadata
metadata := c.get_collection_metadata(collection_name)!
fixed_page_name := texttools.name_fix_no_ext(page_name)
// Find the page in metadata
if fixed_page_name !in metadata.pages {
return error('page_not_found: Page "${page_name}" not found in collection metadata, for collection: "${collection_name}"')
}
page_meta := metadata.pages[fixed_page_name]
// Add all direct links from this page to the result
// This includes: pages, files, images, and external links
all_links << page_meta.links
// Recursively traverse only page-to-page links
for link in page_meta.links {
// Only recursively process links to other pages within the atlas
// Skip external links (http, https, mailto, etc.)
// Skip file and image links (these don't have "contained" links)
if link.file_type != .page || link.status == .external {
continue
}
// Recursively collect links from the target page
c.collect_page_links_recursive(link.target_collection_name, link.target_item_name, mut visited, mut all_links) or {
// If we encounter an error (e.g., target page doesn't exist in metadata),
// we continue processing other links rather than failing completely
// This provides graceful degradation for broken link references
continue
}
}
}
// get_image_links returns all image links found in a page and related pages (recursive)
// This is a convenience function that filters get_page_links to only image links
pub fn (mut c AtlasClient) get_image_links(collection_name string, page_name string) ![]LinkMetadata {
all_links := c.get_page_links(collection_name, page_name)!
mut image_links := []LinkMetadata{}
for link in all_links {
if link.file_type == .image {
image_links << link
}
}
return image_links
}
// get_file_links returns all file links (non-image) found in a page and related pages (recursive)
// This is a convenience function that filters get_page_links to only file links
pub fn (mut c AtlasClient) get_file_links(collection_name string, page_name string) ![]LinkMetadata {
all_links := c.get_page_links(collection_name, page_name)!
mut file_links := []LinkMetadata{}
for link in all_links {
if link.file_type == .file {
file_links << link
}
}
return file_links
}
// get_page_link_targets returns all page-to-page link targets found in a page and related pages
// This is a convenience function that filters get_page_links to only page links
pub fn (mut c AtlasClient) get_page_link_targets(collection_name string, page_name string) ![]LinkMetadata {
all_links := c.get_page_links(collection_name, page_name)!
mut page_links := []LinkMetadata{}
for link in all_links {
if link.file_type == .page && link.status != .external {
page_links << link
}
}
return page_links
}

124
lib/data/atlas/export.v
View File

@@ -7,7 +7,7 @@ import json
@[params]
pub struct ExportArgs {
pub mut:
destination string @[requireds]
destination string @[required]
reset bool = true
include bool = true
redis bool = true
@@ -90,6 +90,44 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! {
c.collect_cross_collection_references(mut page, mut cross_collection_pages, mut
cross_collection_files, mut processed_cross_pages)!
// println('------- ${c.name} ${page.key()}')
// if page.key() == 'geoaware:solution' && c.name == 'mycelium_nodes_tiers' {
// println(cross_collection_pages)
// println(cross_collection_files)
// // println(processed_cross_pages)
// $dbg;
// }
// copy the pages to the right exported path
for _, mut ref_page in cross_collection_pages {
mut src_file := ref_page.path()!
mut subdir_path := pathlib.get_dir(
path: '${col_dir.path}'
create: true
)!
mut dest_path := '${subdir_path.path}/${ref_page.name}.md'
src_file.copy(dest: dest_path)!
// println(dest_path)
// $dbg;
}
// copy the files to the right exported path
for _, mut ref_file in cross_collection_files {
mut src_file2 := ref_file.path()!
// Determine subdirectory based on file type
mut subdir := if ref_file.is_image() { 'img' } else { 'files' }
// Ensure subdirectory exists
mut subdir_path := pathlib.get_dir(
path: '${col_dir.path}/${subdir}'
create: true
)!
mut dest_path := '${subdir_path.path}/${ref_file.name}'
mut dest_file2 := pathlib.get_file(path: dest_path, create: true)!
src_file2.copy(dest: dest_file2.path)!
}
processed_local_pages[page.name] = true
// Redis operations...
@@ -117,65 +155,6 @@ pub fn (mut c Collection) export(args CollectionExportArgs) ! {
mut dest_file := pathlib.get_file(path: dest_path, create: true)!
src_file.copy(dest: dest_file.path)!
}
// Second pass: copy all collected cross-collection pages and process their links recursively
// Keep iterating until no new cross-collection references are found
for {
mut found_new_references := false
// Process all cross-collection pages we haven't processed yet
for page_key, mut ref_page in cross_collection_pages {
if page_key in processed_cross_pages {
continue // Already processed this page's links
}
// Mark as processed to avoid infinite loops
processed_cross_pages[page_key] = true
found_new_references = true
// Get the referenced page content with includes processed
ref_content := ref_page.content_with_fixed_links(
include: args.include
cross_collection: true
export_mode: true
)!
// Write the referenced page to this collection's directory
mut dest_file := pathlib.get_file(
path: '${col_dir.path}/${ref_page.name}.md'
create: true
)!
dest_file.write(ref_content)!
// CRITICAL: Recursively process links in this cross-collection page
// This ensures we get pages/files/images referenced by ref_page
c.collect_cross_collection_references(mut ref_page, mut cross_collection_pages, mut
cross_collection_files, mut processed_cross_pages)!
}
// If we didn't find any new references, we're done with the recursive pass
if !found_new_references {
break
}
}
// Third pass: copy ALL collected cross-collection referenced files/images
for _, mut ref_file in cross_collection_files {
mut src_file := ref_file.path()!
// Determine subdirectory based on file type
mut subdir := if ref_file.is_image() { 'img' } else { 'files' }
// Ensure subdirectory exists
mut subdir_path := pathlib.get_dir(
path: '${col_dir.path}/${subdir}'
create: true
)!
mut dest_path := '${subdir_path.path}/${ref_file.name}'
mut dest_file := pathlib.get_file(path: dest_path, create: true)!
src_file.copy(dest: dest_file.path)!
}
}
// Helper function to recursively collect cross-collection references
@@ -184,6 +163,17 @@ fn (mut c Collection) collect_cross_collection_references(mut page Page,
mut all_cross_pages map[string]&Page,
mut all_cross_files map[string]&File,
mut processed_pages map[string]bool) ! {
page_key := page.key()
// If we've already processed this page, skip it (prevents infinite loops with cycles)
if page_key in processed_pages {
return
}
// Mark this page as processed BEFORE recursing (prevents infinite loops with circular references)
processed_pages[page_key] = true
// Process all links in the current page
// Use cached links from validation (before transformation) to preserve collection info
for mut link in page.links {
if link.status != .found {
@@ -192,15 +182,19 @@ fn (mut c Collection) collect_cross_collection_references(mut page Page,
is_local := link.target_collection_name == c.name
// Collect cross-collection page references
// Collect cross-collection page references and recursively process them
if link.file_type == .page && !is_local {
page_key := '${link.target_collection_name}:${link.target_item_name}'
page_ref := '${link.target_collection_name}:${link.target_item_name}'
// Only add if not already collected
if page_key !in all_cross_pages {
if page_ref !in all_cross_pages {
mut target_page := link.target_page()!
all_cross_pages[page_key] = target_page
// Don't mark as processed yet - we'll do that when we actually process its links
all_cross_pages[page_ref] = target_page
// Recursively process the target page's links to find more cross-collection references
// This ensures we collect ALL transitive cross-collection page and file references
c.collect_cross_collection_references(mut target_page, mut all_cross_pages, mut
all_cross_files, mut processed_pages)!
}
}

15
lib/data/atlas/instruction.md
View File

@@ -1,15 +0,0 @@
in atlas/
check format of groups
see content/groups
now the groups end with .group
check how the include works, so we can include another group in the group as defined, only works in same folder
in the scan function in atlas, now make scan_groups function, find groups, only do this for collection as named groups
do not add collection groups to atlas, this is a system collection
make the groups and add them to atlas
give clear instructions for coding agent how to write the code

3
lib/data/atlas/play.v
View File

@@ -3,6 +3,7 @@ module atlas
import incubaid.herolib.core.playbook { PlayBook }
import incubaid.herolib.develop.gittools
import incubaid.herolib.ui.console
import os
// Play function to process HeroScript actions for Atlas
pub fn play(mut plbook PlayBook) ! {
@@ -66,7 +67,7 @@ pub fn play(mut plbook PlayBook) ! {
for mut action in export_actions {
mut p := action.params
name = p.get_default('name', 'main')!
destination := p.get_default('destination', '/tmp/atlas_export')!
destination := p.get_default('destination', '${os.home_dir()}/hero/var/atlas_export')!
reset := p.get_default_true('reset')
include := p.get_default_true('include')
redis := p.get_default_true('redis')

4
lib/data/atlas/process.md
View File

@@ -1,4 +0,0 @@
- first find all pages
- then for each page find all links

2
lib/data/atlas/readme.md
View File

@@ -33,7 +33,7 @@ put in .hero file and execute with hero or but shebang line on top of .hero scri
!!atlas.scan git_url:"https://git.ourworld.tf/tfgrid/docs_tfgrid4/src/branch/main/collections/tests"
!!atlas.export destination: '/tmp/atlas_export'
!!atlas.export
```

6
lib/web/docusaurus/dsite.v
View File

@@ -71,9 +71,9 @@ pub struct DevArgs {
pub mut:
host string = 'localhost'
port int = 3000
open bool = true // whether to open the browser automatically
watch_changes bool = false // whether to watch for changes in docs and rebuild automatically
skip_generate bool = false // whether to skip generation (useful when docs are pre-generated, e.g., from atlas)
open bool = true // whether to open the browser automatically
watch_changes bool // whether to watch for changes in docs and rebuild automatically
skip_generate bool // whether to skip generation (useful when docs are pre-generated, e.g., from atlas)
}
pub fn (mut s DocSite) open(args DevArgs) ! {

4
lib/web/docusaurus/dsite_generate_docs.v
View File

@@ -142,6 +142,10 @@ fn (mut generator SiteGenerator) page_generate(args_ Page) ! {
pagefile.write(c)!
generator.client.copy_pages(collection_name, page_name, pagefile.path_dir()) or {
generator.error("Couldn't copy pages for page:'${page_name}' in collection:'${collection_name}'\nERROR:${err}")!
return
}
generator.client.copy_images(collection_name, page_name, pagefile.path_dir()) or {
generator.error("Couldn't copy images for page:'${page_name}' in collection:'${collection_name}'\nERROR:${err}")!
return

30
lib/web/docusaurus/interface_atlas_client.v
View File

@@ -1,30 +0,0 @@
module docusaurus
pub interface IDocClient {
mut:
// Path methods - get absolute paths to resources
get_page_path(collection_name string, page_name string) !string
get_file_path(collection_name string, file_name string) !string
get_image_path(collection_name string, image_name string) !string
// Existence checks - verify if resources exist
page_exists(collection_name string, page_name string) bool
file_exists(collection_name string, file_name string) bool
image_exists(collection_name string, image_name string) bool
// Content retrieval
get_page_content(collection_name string, page_name string) !string
// Listing methods - enumerate resources
list_collections() ![]string
list_pages(collection_name string) ![]string
list_files(collection_name string) ![]string
list_images(collection_name string) ![]string
list_pages_map() !map[string][]string
list_markdown() !string
// Image operations
// get_page_paths(collection_name string, page_name string) !(string, []string)
copy_images(collection_name string, page_name string, destination_path string) !
copy_files(collection_name string, page_name string, destination_path string) !
}

3
lib/web/docusaurus/play.v
View File

@@ -1,6 +1,7 @@
module docusaurus
import incubaid.herolib.core.playbook { PlayBook }
import os
pub fn play(mut plbook PlayBook) ! {
if !plbook.exists(filter: 'docusaurus.') {
@@ -17,7 +18,7 @@ pub fn play(mut plbook PlayBook) ! {
reset: param_define.get_default_false('reset')
template_update: param_define.get_default_false('template_update')
install: param_define.get_default_false('install')
atlas_dir: param_define.get_default('atlas_dir', '/tmp/atlas_export')!
atlas_dir: param_define.get_default('atlas_dir', '${os.home_dir()}/hero/var/atlas_export')!
use_atlas: param_define.get_default_false('use_atlas')
)!

536
lib/web/site/ai_instructions.md
View File

@@ -1,536 +0,0 @@
# AI Instructions for Site Module HeroScript
This document provides comprehensive instructions for AI agents working with the Site module's HeroScript format.
## HeroScript Format Overview
HeroScript is a declarative configuration language with the following characteristics:
### Basic Syntax
```heroscript
!!actor.action
param1: "value1"
param2: "value2"
multiline_param: "
This is a multiline value.
It can span multiple lines.
"
arg1 arg2 // Arguments without keys
```
**Key Rules:**
1. Actions start with `!!` followed by `actor.action` format
2. Parameters are indented and use `key: "value"` or `key: value` format
3. Values with spaces must be quoted
4. Multiline values are supported with quotes
5. Arguments without keys are space-separated
6. Comments start with `//`
## Site Module Actions
### 1. Site Configuration (`!!site.config`)
**Purpose:** Define the main site configuration including title, description, and metadata.
**Required Parameters:**
- `name`: Site identifier (will be normalized to snake_case)
**Optional Parameters:**
- `title`: Site title (default: "Documentation Site")
- `description`: Site description
- `tagline`: Site tagline
- `favicon`: Path to favicon (default: "img/favicon.png")
- `image`: Default site image (default: "img/tf_graph.png")
- `copyright`: Copyright text
- `url`: Main site URL
- `base_url`: Base URL path (default: "/")
- `url_home`: Home page path
**Example:**
```heroscript
!!site.config
name: "my_documentation"
title: "My Documentation Site"
description: "Comprehensive technical documentation"
tagline: "Learn everything you need"
url: "https://docs.example.com"
base_url: "/"
```
**AI Guidelines:**
- Always include `name` parameter
- Use descriptive titles and descriptions
- Ensure URLs are properly formatted with protocol
### 2. Metadata Configuration (`!!site.config_meta`)
**Purpose:** Override specific metadata for SEO purposes.
**Optional Parameters:**
- `title`: SEO-specific title (overrides site.config title for meta tags)
- `image`: SEO-specific image (overrides site.config image for og:image)
- `description`: SEO-specific description
**Example:**
```heroscript
!!site.config_meta
title: "My Docs - Complete Guide"
image: "img/social-preview.png"
description: "The ultimate guide to using our platform"
```
**AI Guidelines:**
- Use only when SEO metadata needs to differ from main config
- Keep titles concise for social media sharing
- Use high-quality images for social previews
### 3. Navigation Bar (`!!site.navbar` or `!!site.menu`)
**Purpose:** Configure the main navigation bar.
**Optional Parameters:**
- `title`: Navigation title (defaults to site.config title)
- `logo_alt`: Logo alt text
- `logo_src`: Logo image path
- `logo_src_dark`: Dark mode logo path
**Example:**
```heroscript
!!site.navbar
title: "My Site"
logo_alt: "My Site Logo"
logo_src: "img/logo.svg"
logo_src_dark: "img/logo-dark.svg"
```
**AI Guidelines:**
- Use `!!site.navbar` for modern syntax (preferred)
- `!!site.menu` is supported for backward compatibility
- Provide both light and dark logos when possible
### 4. Navigation Items (`!!site.navbar_item` or `!!site.menu_item`)
**Purpose:** Add items to the navigation bar.
**Required Parameters (one of):**
- `to`: Internal link path
- `href`: External URL
**Optional Parameters:**
- `label`: Display text (required in practice)
- `position`: "left" or "right" (default: "right")
**Example:**
```heroscript
!!site.navbar_item
label: "Documentation"
to: "docs/intro"
position: "left"
!!site.navbar_item
label: "GitHub"
href: "https://github.com/myorg/repo"
position: "right"
```
**AI Guidelines:**
- Use `to` for internal navigation
- Use `href` for external links
- Position important items on the left, secondary items on the right
### 5. Footer Configuration (`!!site.footer`)
**Purpose:** Configure footer styling.
**Optional Parameters:**
- `style`: "dark" or "light" (default: "dark")
**Example:**
```heroscript
!!site.footer
style: "dark"
```
### 6. Footer Items (`!!site.footer_item`)
**Purpose:** Add links to the footer, grouped by title.
**Required Parameters:**
- `title`: Group title (items with same title are grouped together)
- `label`: Link text
**Required Parameters (one of):**
- `to`: Internal link path
- `href`: External URL
**Example:**
```heroscript
!!site.footer_item
title: "Docs"
label: "Introduction"
to: "intro"
!!site.footer_item
title: "Docs"
label: "API Reference"
to: "api"
!!site.footer_item
title: "Community"
label: "Discord"
href: "https://discord.gg/example"
```
**AI Guidelines:**
- Group related links under the same title
- Use consistent title names across related items
- Provide both internal and external links as appropriate
### 7. Page Categories (`!!site.page_category`)
**Purpose:** Create a section/category to organize pages.
**Required Parameters:**
- `name`: Category identifier (snake_case)
**Optional Parameters:**
- `label`: Display name (auto-generated from name if not provided)
- `position`: Manual sort order (auto-incremented if not specified)
- `path`: URL path segment (defaults to normalized label)
**Example:**
```heroscript
!!site.page_category
name: "getting_started"
label: "Getting Started"
position: 100
!!site.page_category
name: "advanced_topics"
label: "Advanced Topics"
```
**AI Guidelines:**
- Use descriptive snake_case names
- Let label be auto-generated when possible (name_fix converts to Title Case)
- Categories persist for all subsequent pages until a new category is declared
- Position values should leave gaps (100, 200, 300) for future insertions
### 8. Pages (`!!site.page`)
**Purpose:** Define individual pages in the site.
**Required Parameters:**
- `src`: Source reference as `collection:page_name` (required for first page in a collection)
**Optional Parameters:**
- `name`: Page identifier (extracted from src if not provided)
- `title`: Page title (extracted from markdown if not provided)
- `description`: Page description for metadata
- `slug`: Custom URL slug
- `position`: Manual sort order (auto-incremented if not specified)
- `draft`: Mark as draft (default: false)
- `hide_title`: Hide title in rendering (default: false)
- `path`: Custom path (defaults to current category name)
- `category`: Override current category
- `title_nr`: Title numbering level
**Example:**
```heroscript
!!site.page src: "docs:introduction"
description: "Introduction to the platform"
slug: "/"
!!site.page src: "quickstart"
description: "Get started in 5 minutes"
!!site.page src: "installation"
title: "Installation Guide"
description: "How to install and configure"
position: 10
```
**AI Guidelines:**
- **Collection Persistence:** Specify collection once (e.g., `docs:introduction`), then subsequent pages only need page name (e.g., `quickstart`)
- **Category Persistence:** Pages belong to the most recently declared category
- **Title Extraction:** Prefer extracting titles from markdown files
- **Position Management:** Use automatic positioning unless specific order is required
- **Description Required:** Always provide descriptions for SEO
- **Slug Usage:** Use slug for special pages like homepage (`slug: "/"`)
### 9. Import External Content (`!!site.import`)
**Purpose:** Import content from external sources.
**Optional Parameters:**
- `name`: Import identifier
- `url`: Git URL or HTTP URL
- `path`: Local file system path
- `dest`: Destination path in site
- `replace`: Comma-separated key:value pairs for variable replacement
- `visible`: Whether imported content is visible (default: true)
**Example:**
```heroscript
!!site.import
url: "https://github.com/example/docs"
dest: "external"
replace: "VERSION:1.0.0,PROJECT:MyProject"
visible: true
```
**AI Guidelines:**
- Use for shared documentation across multiple sites
- Replace variables using `${VARIABLE}` syntax in source content
- Set `visible: false` for imported templates or partials
### 10. Publish Destinations (`!!site.publish` and `!!site.publish_dev`)
**Purpose:** Define where to publish the built site.
**Optional Parameters:**
- `path`: File system path or URL
- `ssh_name`: SSH connection name for remote deployment
**Example:**
```heroscript
!!site.publish
path: "/var/www/html/docs"
ssh_name: "production_server"
!!site.publish_dev
path: "/tmp/docs-preview"
```
**AI Guidelines:**
- Use `!!site.publish` for production deployments
- Use `!!site.publish_dev` for development/preview deployments
- Can specify multiple destinations
## File Organization Best Practices
### Naming Convention
Use numeric prefixes to control execution order:
```
0_config.heroscript # Site configuration
1_navigation.heroscript # Menu and footer
2_intro.heroscript # Introduction pages
3_guides.heroscript # User guides
4_reference.heroscript # API reference
```
**AI Guidelines:**
- Always use numeric prefixes (0_, 1_, 2_, etc.)
- Leave gaps in numbering (0, 10, 20) for future insertions
- Group related configurations in the same file
- Process order matters: config → navigation → pages
### Execution Order Rules
1. **Configuration First:** `!!site.config` must be processed before other actions
2. **Categories Before Pages:** Declare `!!site.page_category` before pages in that category
3. **Collection Persistence:** First page in a collection must specify `collection:page_name`
4. **Category Persistence:** Pages inherit the most recent category declaration
## Common Patterns
### Pattern 1: Simple Documentation Site
```heroscript
!!site.config
name: "simple_docs"
title: "Simple Documentation"
!!site.navbar
title: "Simple Docs"
!!site.page src: "docs:index"
description: "Welcome page"
slug: "/"
!!site.page src: "getting-started"
description: "Getting started guide"
!!site.page src: "api"
description: "API reference"
```
### Pattern 2: Multi-Section Documentation
```heroscript
!!site.config
name: "multi_section_docs"
title: "Complete Documentation"
!!site.page_category
name: "introduction"
label: "Introduction"
!!site.page src: "docs:welcome"
description: "Welcome to our documentation"
!!site.page src: "overview"
description: "Platform overview"
!!site.page_category
name: "tutorials"
label: "Tutorials"
!!site.page src: "tutorial_basics"
description: "Basic tutorial"
!!site.page src: "tutorial_advanced"
description: "Advanced tutorial"
```
### Pattern 3: Complex Site with External Links
```heroscript
!!site.config
name: "complex_site"
title: "Complex Documentation Site"
url: "https://docs.example.com"
!!site.navbar
title: "My Platform"
logo_src: "img/logo.svg"
!!site.navbar_item
label: "Docs"
to: "docs/intro"
position: "left"
!!site.navbar_item
label: "API"
to: "api"
position: "left"
!!site.navbar_item
label: "GitHub"
href: "https://github.com/example/repo"
position: "right"
!!site.footer
style: "dark"
!!site.footer_item
title: "Documentation"
label: "Getting Started"
to: "docs/intro"
!!site.footer_item
title: "Community"
label: "Discord"
href: "https://discord.gg/example"
!!site.page_category
name: "getting_started"
!!site.page src: "docs:introduction"
description: "Introduction to the platform"
slug: "/"
!!site.page src: "installation"
description: "Installation guide"
```
## Error Prevention
### Common Mistakes to Avoid
1. **Missing Collection on First Page:**
```heroscript
# WRONG - no collection specified
!!site.page src: "introduction"
# CORRECT
!!site.page src: "docs:introduction"
```
2. **Category Without Name:**
```heroscript
# WRONG - missing name
!!site.page_category
label: "Getting Started"
# CORRECT
!!site.page_category
name: "getting_started"
label: "Getting Started"
```
3. **Missing Description:**
```heroscript
# WRONG - no description
!!site.page src: "docs:intro"
# CORRECT
!!site.page src: "docs:intro"
description: "Introduction to the platform"
```
4. **Incorrect File Ordering:**
```
# WRONG - pages before config
pages.heroscript
config.heroscript
# CORRECT - config first
0_config.heroscript
1_pages.heroscript
```
## Validation Checklist
When generating HeroScript for the Site module, verify:
- [ ] `!!site.config` includes `name` parameter
- [ ] All pages have `description` parameter
- [ ] First page in each collection specifies `collection:page_name`
- [ ] Categories are declared before their pages
- [ ] Files use numeric prefixes for ordering
- [ ] Navigation items have either `to` or `href`
- [ ] Footer items are grouped by `title`
- [ ] External URLs include protocol (https://)
- [ ] Paths don't have trailing slashes unless intentional
- [ ] Draft pages are marked with `draft: true`
## Integration with V Code
When working with the Site module in V code:
```v
import incubaid.herolib.web.site
import incubaid.herolib.core.playbook
// Process HeroScript files
mut plbook := playbook.new(path: '/path/to/heroscripts')!
site.play(mut plbook)!
// Access configured site
mut mysite := site.get(name: 'my_site')!
// Iterate through pages
for page in mysite.pages {
println('Page: ${page.name} - ${page.description}')
}
// Iterate through sections
for section in mysite.sections {
println('Section: ${section.label}')
}
```
## Summary
The Site module's HeroScript format provides a declarative way to configure websites with:
- Clear separation of concerns (config, navigation, content)
- Automatic ordering and organization
- Collection and category persistence for reduced repetition
- Flexible metadata and SEO configuration
- Support for both internal and external content
Always follow the execution order rules, use numeric file prefixes, and provide complete metadata for best results.

145
lib/web/site/model_nav.v Normal file
View File

@@ -0,0 +1,145 @@
module site
import json
// Top-level config
pub struct NavConfig {
pub mut:
mySidebar []NavItem
// myTopbar []NavItem //not used yet
// myFooter []NavItem //not used yet
}
// -------- Variant Type --------
pub type NavItem = NavDoc | NavCat | NavLink
// --------- DOC ITEM ----------
pub struct NavDoc {
pub:
id string //is the page id
label string
}
// --------- CATEGORY ----------
pub struct NavCat {
pub mut:
label string
collapsible bool
collapsed bool
items []NavItem
}
// --------- LINK ----------
pub struct NavLink {
pub:
label string
href string
description string
}
// -------- JSON SERIALIZATION --------
// NavItemJson is used for JSON export with type discrimination
pub struct NavItemJson {
pub mut:
type_field string @[json: 'type']
// For doc
id string @[omitempty]
label string @[omitempty]
// For link
href string @[omitempty]
description string @[omitempty]
// For category
collapsible bool
collapsed bool
items []NavItemJson @[omitempty]
}
// Convert a single NavItem to JSON-serializable format
fn nav_item_to_json(item NavItem) !NavItemJson {
return match item {
NavDoc {
NavItemJson{
type_field: 'doc'
id: item.id
label: item.label
collapsible: false
collapsed: false
}
}
NavLink {
NavItemJson{
type_field: 'link'
label: item.label
href: item.href
description: item.description
collapsible: false
collapsed: false
}
}
NavCat {
mut json_items := []NavItemJson{}
for sub_item in item.items {
json_items << nav_item_to_json(sub_item)!
}
NavItemJson{
type_field: 'category'
label: item.label
collapsible: item.collapsible
collapsed: item.collapsed
items: json_items
}
}
}
}
// Convert entire NavConfig sidebar to JSON-serializable array
fn (nc NavConfig) sidebar_to_json() ![]NavItemJson {
mut result := []NavItemJson{}
for item in nc.mySidebar {
result << nav_item_to_json(item)!
}
return result
}
// // Convert entire NavConfig topbar to JSON-serializable array
// fn (nc NavConfig) topbar_to_json() ![]NavItemJson {
// mut result := []NavItemJson{}
// for item in nc.myTopbar {
// result << nav_item_to_json(item)!
// }
// return result
// }
// // Convert entire NavConfig footer to JSON-serializable array
// fn (nc NavConfig) footer_to_json() ![]NavItemJson {
// mut result := []NavItemJson{}
// for item in nc.myFooter {
// result << nav_item_to_json(item)!
// }
// return result
// }
port topbar as formatted JSON string
// pub fn (nc NavConfig) jsondump_topbar() !string {
// items := nc.topbar_to_json()!
// return json.encode_pretty(items)
// }
// // Export footer as formatted JSON string
// pub fn (nc NavConfig) jsondump_footer() !string {
// items := nc.footer_to_json()!
// return json.encode_pretty(items)
// }
// // Export all navigation as object with sidebar, topbar, footer
// pub fn (nc NavConfig) jsondump_all() !string {
// all_nav := map[string][]NavItemJson{
// 'sidebar': nc.sidebar_to_json()!
// 'topbar': nc.topbar_to_json()!
// 'footer': nc.footer_to_json()!
// }
// return json.encode_pretty(all_nav)
// }

7
lib/web/site/model_page.v
View File

@@ -2,15 +2,10 @@ module site
pub struct Page {
pub mut:
name string
id string //unique identifier, by default as "collection:page_name", we can overrule this from play instructions if needed
title string
description string
draft bool
position int
hide_title bool
src string @[required] // always in format collection:page_name, can use the default collection if no : specified
path string @[required] // is without the page name, so just the path to the folder where the page is in
section_name string
title_nr int
slug string
}

9
lib/web/site/model_site.v Normal file
View File

@@ -0,0 +1,9 @@
module site
@[heap]
pub struct Site {
pub mut:
pages map[string]Page // key: is the id of the page
nav NavConfig //navigation of the site
siteconfig SiteConfig
}

18
lib/web/site/model_site_section.v
View File

@@ -1,18 +0,0 @@
module site
@[heap]
pub struct Site {
pub mut:
pages []Page
sections []Section
siteconfig SiteConfig
}
pub struct Section {
pub mut:
name string
position int
path string
label string
description string
}