MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

...

Browse Source
This commit is contained in:
despiegk
2025-10-16 09:25:03 +04:00
parent 6918a02eff
commit cf601283b1
10 changed files with 821 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
lib/data/atlas/atlas.v Normal file
View File

@@ -0,0 +1,101 @@
module atlas
import incubaid.herolib.core.texttools
import incubaid.herolib.core.pathlib
__global (
atlases shared map[string]&Atlas
)
@[heap]
pub struct Atlas {
pub mut:
name string
collections map[string]&Collection
}
@[params]
pub struct AtlasNewArgs {
pub mut:
name string = 'default'
}
// Create a new Atlas
pub fn new(args AtlasNewArgs) !&Atlas {
mut name := texttools.name_fix(args.name)
mut a := Atlas{
name: name
}
atlas_set(a)
return &a
}
// Get Atlas from global map
pub fn atlas_get(name string) !&Atlas {
rlock atlases {
if name in atlases {
return atlases[name] or { return error('Atlas ${name} not found') }
}
}
return error("Atlas '${name}' not found")
}
// Check if Atlas exists
pub fn atlas_exists(name string) bool {
rlock atlases {
return name in atlases
}
}
// List all Atlas names
pub fn atlas_list() []string {
rlock atlases {
return atlases.keys()
}
}
// Store Atlas in global map
fn atlas_set(atlas Atlas) {
lock atlases {
atlases[atlas.name] = &atlas
}
}
@[params]
pub struct AddCollectionArgs {
pub mut:
name string @[required]
path string @[required]
}
// Add a collection to the Atlas
pub fn (mut a Atlas) add_collection(args AddCollectionArgs) ! {
name := texttools.name_fix(args.name)
if name in a.collections {
return error('Collection ${name} already exists in Atlas ${a.name}')
}
mut col := a.new_collection(name: name, path: args.path)!
col.scan()!
a.collections[name] = &col
}
// Scan a path for collections
pub fn (mut a Atlas) scan(args ScanArgs) ! {
mut path := pathlib.get_dir(path: args.path)!
a.scan_directory(mut path)!
}
// Get a collection by name
pub fn (a Atlas) get_collection(name string) !&Collection {
return a.collections[name] or {
return CollectionNotFound{
name: name
msg: 'Collection not found in Atlas ${a.name}'
}
}
}

76
lib/data/atlas/atlas_test.v Normal file
View File

@@ -0,0 +1,76 @@
module atlas
import incubaid.herolib.core.pathlib
import os
const test_base = '/tmp/atlas_test'
fn testsuite_begin() {
os.rmdir_all(test_base) or {}
os.mkdir_all(test_base)!
}
fn testsuite_end() {
os.rmdir_all(test_base) or {}
}
fn test_create_atlas() {
mut a := new(name: 'test_atlas')!
assert a.name == 'test_atlas'
assert a.collections.len == 0
}
fn test_add_collection() {
// Create test collection
col_path := '${test_base}/col1'
os.mkdir_all(col_path)!
mut cfile := pathlib.get_file(path: '${col_path}/.collection', create: true)!
cfile.write('name:col1')!
mut page := pathlib.get_file(path: '${col_path}/page1.md', create: true)!
page.write('# Page 1\n\nContent here.')!
mut a := new(name: 'test')!
a.add_collection(name: 'col1', path: col_path)!
assert a.collections.len == 1
assert 'col1' in a.collections
}
fn test_scan() {
// Create test structure
os.mkdir_all('${test_base}/docs/guides')!
mut cfile := pathlib.get_file(path: '${test_base}/docs/guides/.collection', create: true)!
cfile.write('name:guides')!
mut page := pathlib.get_file(path: '${test_base}/docs/guides/intro.md', create: true)!
page.write('# Introduction')!
mut a := new()!
a.scan(path: '${test_base}/docs')!
assert a.collections.len == 1
col := a.get_collection('guides')!
assert col.page_exists('intro')
}
fn test_export() {
// Setup
col_path := '${test_base}/source/col1'
export_path := '${test_base}/export'
os.mkdir_all(col_path)!
mut cfile := pathlib.get_file(path: '${col_path}/.collection', create: true)!
cfile.write('name:col1')!
mut page := pathlib.get_file(path: '${col_path}/test.md', create: true)!
page.write('# Test Page')!
mut a := new()!
a.add_collection(name: 'col1', path: col_path)!
a.export(destination: export_path, redis: false)!
assert os.exists('${export_path}/col1/test.md')
assert os.exists('${export_path}/col1/.collection')
}

116
lib/data/atlas/collection.v Normal file
View File

@@ -0,0 +1,116 @@
module atlas
import incubaid.herolib.core.pathlib
import incubaid.herolib.core.texttools
@[heap]
pub struct Collection {
pub mut:
name string @[required]
path pathlib.Path @[required]
pages map[string]&Page
images map[string]&File
files map[string]&File
atlas &Atlas @[skip]
}
@[params]
pub struct CollectionNewArgs {
pub mut:
name string @[required]
path string @[required]
}
// Create a new collection
fn (mut self Atlas) new_collection(args CollectionNewArgs) !Collection {
mut name := texttools.name_fix(args.name)
mut path := pathlib.get_dir(path: args.path)!
mut col := Collection{
name: name
path: path
atlas: &self
}
return col
}
// Add a page to the collection
fn (mut c Collection) add_page(mut p pathlib.Path) ! {
name := p.name_fix_no_ext()
if name in c.pages {
return error('Page ${name} already exists in collection ${c.name}')
}
p_new := new_page(
name: name
path: p
collection_name: c.name
)!
c.pages[name] = &p_new
}
// Add an image to the collection
fn (mut c Collection) add_image(mut p pathlib.Path) ! {
name := p.name_fix_no_ext()
if name in c.images {
return error('Image ${name} already exists in collection ${c.name}')
}
mut img := new_file(path: p)!
c.images[name] = &img
}
// Add a file to the collection
fn (mut c Collection) add_file(mut p pathlib.Path) ! {
name := p.name_fix_no_ext()
if name in c.files {
return error('File ${name} already exists in collection ${c.name}')
}
mut file := new_file(path: p)!
c.files[name] = &file
}
// Get a page by name
pub fn (c Collection) page_get(name string) !&Page {
return c.pages[name] or { return PageNotFound{
collection: c.name
page: name
} }
}
// Get an image by name
pub fn (c Collection) image_get(name string) !&File {
return c.images[name] or { return FileNotFound{
collection: c.name
file: name
} }
}
// Get a file by name
pub fn (c Collection) file_get(name string) !&File {
return c.files[name] or { return FileNotFound{
collection: c.name
file: name
} }
}
// Check if page exists
pub fn (c Collection) page_exists(name string) bool {
return name in c.pages
}
// Check if image exists
pub fn (c Collection) image_exists(name string) bool {
return name in c.images
}
// Check if file exists
pub fn (c Collection) file_exists(name string) bool {
return name in c.files
}

34
lib/data/atlas/error.v Normal file
View File

@@ -0,0 +1,34 @@
module atlas
pub struct CollectionNotFound {
Error
pub:
name string
msg string
}
pub fn (err CollectionNotFound) msg() string {
return 'Collection ${err.name} not found: ${err.msg}'
}
pub struct PageNotFound {
Error
pub:
collection string
page string
}
pub fn (err PageNotFound) msg() string {
return 'Page ${err.page} not found in collection ${err.collection}'
}
pub struct FileNotFound {
Error
pub:
collection string
file string
}
pub fn (err FileNotFound) msg() string {
return 'File ${err.file} not found in collection ${err.collection}'
}

118
lib/data/atlas/export.v Normal file
View File

@@ -0,0 +1,118 @@
module atlas
import incubaid.herolib.core.pathlib
import incubaid.herolib.core.base
import os
@[params]
pub struct ExportArgs {
pub mut:
destination string
reset bool = true
redis bool = true
}
// Export all collections
pub fn (mut a Atlas) export(args ExportArgs) ! {
mut dest := pathlib.get_dir(path: args.destination, create: true)!
if args.reset {
dest.empty()!
}
for _, mut col in a.collections {
col.export(
destination: dest
reset: args.reset
redis: args.redis
)!
}
}
@[params]
pub struct CollectionExportArgs {
pub mut:
destination pathlib.Path @[required]
reset bool = true
redis bool = true
}
// Export a single collection
pub fn (mut c Collection) export(args CollectionExportArgs) ! {
// Create collection directory
col_dir := pathlib.get_dir(
path: '${args.destination.path}/${c.name}'
create: true
)!
// Write .collection file
mut cfile := pathlib.get_file(
path: '${col_dir.path}/.collection'
create: true
)!
cfile.write("name:${c.name} src:'${c.path.path}'")!
// Export pages
export_pages(c.name, c.pages.values(), col_dir, args.redis)!
// Export images
export_files(c.name, c.images.values(), col_dir, 'img', args.redis)!
// Export files
export_files(c.name, c.files.values(), col_dir, 'files', args.redis)!
// Store collection metadata in Redis if enabled
if args.redis {
mut context := base.context()!
mut redis := context.redis()!
redis.hset('atlas:path', c.name, col_dir.path)!
}
}
// Export pages to destination
fn export_pages(col_name string, pages []&Page, dest pathlib.Path, redis bool) ! {
mut context := base.context()!
mut redis_client := context.redis()!
for mut page in pages {
// Simple copy of markdown content
content := page.read_content()!
mut dest_file := pathlib.get_file(
path: '${dest.path}/${page.name}.md'
create: true
)!
dest_file.write(content)!
if redis {
redis_client.hset('atlas:${col_name}', page.name, '${page.name}.md')!
}
}
}
// Export files/images to destination
fn export_files(col_name string, files []&File, dest pathlib.Path, subdir string, redis bool) ! {
if files.len == 0 {
return
}
mut context := base.context()!
mut redis_client := context.redis()!
// Create subdirectory
files_dir := pathlib.get_dir(
path: '${dest.path}/${subdir}'
create: true
)!
for mut file in files {
dest_path := '${files_dir.path}/${file.file_name()}'
// Copy file
file.path.copy(dest: dest_path)!
if redis {
redis_client.hset('atlas:${col_name}', file.file_name(), '${subdir}/${file.file_name()}')!
}
}
}

51
lib/data/atlas/file.v Normal file
View File

@@ -0,0 +1,51 @@
module atlas
import incubaid.herolib.core.pathlib
pub enum FileType {
file
image
}
pub struct File {
pub mut:
name string // name without extension
ext string // file extension
path pathlib.Path // full path to file
ftype FileType // file or image
}
@[params]
pub struct NewFileArgs {
pub:
path pathlib.Path @[required]
}
pub fn new_file(args NewFileArgs) !File {
mut f := File{
path: args.path
}
f.init()!
return f
}
fn (mut f File) init() ! {
// Determine file type
if f.path.is_image() {
f.ftype = .image
} else {
f.ftype = .file
}
// Extract name and extension
f.name = f.path.name_fix_no_ext()
f.ext = f.path.extension_lower()
}
pub fn (f File) file_name() string {
return '${f.name}.${f.ext}'
}
pub fn (f File) is_image() bool {
return f.ftype == .image
}

83
lib/data/atlas/getters.v Normal file
View File

@@ -0,0 +1,83 @@
module atlas
// Get a page from any collection using format "collection:page"
pub fn (a Atlas) page_get(key string) !&Page {
parts := key.split(':')
if parts.len != 2 {
return error('Invalid page key format. Use "collection:page"')
}
col := a.get_collection(parts[0])!
return col.page_get(parts[1])!
}
// Get an image from any collection using format "collection:image"
pub fn (a Atlas) image_get(key string) !&File {
parts := key.split(':')
if parts.len != 2 {
return error('Invalid image key format. Use "collection:image"')
}
col := a.get_collection(parts[0])!
return col.image_get(parts[1])!
}
// Get a file from any collection using format "collection:file"
pub fn (a Atlas) file_get(key string) !&File {
parts := key.split(':')
if parts.len != 2 {
return error('Invalid file key format. Use "collection:file"')
}
col := a.get_collection(parts[0])!
return col.file_get(parts[1])!
}
// Check if page exists
pub fn (a Atlas) page_exists(key string) bool {
parts := key.split(':')
if parts.len != 2 {
return false
}
col := a.get_collection(parts[0]) or { return false }
return col.page_exists(parts[1])
}
// Check if image exists
pub fn (a Atlas) image_exists(key string) bool {
parts := key.split(':')
if parts.len != 2 {
return false
}
col := a.get_collection(parts[0]) or { return false }
return col.image_exists(parts[1])
}
// Check if file exists
pub fn (a Atlas) file_exists(key string) bool {
parts := key.split(':')
if parts.len != 2 {
return false
}
col := a.get_collection(parts[0]) or { return false }
return col.file_exists(parts[1])
}
// List all pages in Atlas
pub fn (a Atlas) list_pages() map[string][]string {
mut result := map[string][]string{}
for col_name, col in a.collections {
mut page_names := []string{}
for page_name, _ in col.pages {
page_names << page_name
}
page_names.sort()
result[col_name] = page_names
}
return result
}

35
lib/data/atlas/page.v Normal file
View File

@@ -0,0 +1,35 @@
module atlas
import incubaid.herolib.core.pathlib
pub struct Page {
pub mut:
name string // name without extension
path pathlib.Path // full path to markdown file
collection_name string // parent collection name
}
@[params]
pub struct NewPageArgs {
pub:
name string @[required]
path pathlib.Path @[required]
collection_name string @[required]
}
pub fn new_page(args NewPageArgs) !Page {
return Page{
name: args.name
path: args.path
collection_name: args.collection_name
}
}
// Simple content reading (no processing)
pub fn (mut p Page) read_content() !string {
return p.path.read()!
}
pub fn (p Page) key() string {
return '${p.collection_name}:${p.name}'
}

162
lib/data/atlas/readme.md
View File

@@ -1,61 +1,107 @@
atlas is a tool which walks over directories, reads metadata files and generates a site structure.
# Atlas Module
specs
A lightweight document collection manager for V, inspired by doctree but simplified.
- walk over directories recursively (use path module)
- find .collection files, each of them defines a collection
- for each collection rename .collection to .collection.json
- init .collection.json with default values if not present
- this is just intial step to get started
- create Atlas struct which holds all collections
- key is collection name, value is Collection struct
- each Collection struct has name, path, pages map, files map
- make a factory to create or get an Atlas struct
- make a function to load an atlas from a given path
- now find .collection.json files and read them into a Collection struct
- find all files with .md extension as well as other files (images and other files)
- remember these files per collection in a Collection struct
- keep a dict in a collection for pages
- the key is a texttools namefix of the filename without extension, the value is a Page struct
- keep a dict in a collection for other files
- the key is the filename, the value is a File struct
- make a save() function on collection which saves the collection as a json file .collection.json in the collection directory
- make a find_page(collection_name, page_name) function on Atlas which returns a Page struct or error
- make a find_file(collection_name, file_name) function on Atlas which returns a File struct
- make a list_collections() function on Atlas which returns a list of collection names
- make a list_pages(collection_name) function on Atlas which returns a list of page names in that collection
- make a list_files(collection_name) function on Atlas which returns a list of file names in that collection
- make a function to add or update a page in a collection
- this function takes collection name, page name, title, description, draft status, position
- it updates or adds the page in the collection's pages dict
- it saves the collection afterwards
- make a function to add or update a file in a collection
- this function takes collection name, file name, path
- it updates or adds the file in the collection's files dict
- it saves the collection afterwards
- make a function to delete a page from a collection
- this function takes collection name, page name
- it removes the page from the collection's pages dict
- it saves the collection afterwards
- make a function to delete a file from a collection
- this function takes collection name, file name
- it removes the file from the collection's files dict
- it saves the collection afterwards
- create a link_check function on page, which checks if all links in the page content are valid
- it uses the Atlas struct to check if linked pages or files exist
- links can be in the form of collection_name:page_name or collection_name:file_name
- if collection_name is omitted, it is assumed to be the current collection
- it can also be http... links which are ignored in the check
- if paths ignore the leading / or ./ or ../ as well as path part, only focus on the last part (the name)
- do namefix on names before checking
- it creates error objects in collection
- it returns a markdown file where links are replaced to:
- collection:page_name if valid page in other collection
- relative path in the collection if valid page in same collection (relative from page where link is found)
- if error we just leave original link
- create a list of Error objects on Collection so we know what is wrong with a collection
- errors can be missing .collection.json, invalid json, missing title in page, broken links in pages
- create a validate() function on Collection which checks for errors and fills the errors list
- create a validate() function on Atlas which validates all collections
- create a report() function on Atlas which prints a report of all collections and their errors as markdown
## Features
- **Simple Collection Scanning**: Automatically find collections marked with `.collection` files
- **Minimal Processing**: No markdown parsing, includes, or link resolution
- **Easy Export**: Copy files to destination with simple organization
- **Optional Redis**: Store metadata in Redis for quick lookups
- **Type-Safe Access**: Get pages, images, and files with error handling
## Quick Start
```v
import incubaid.herolib.data.atlas
// Create a new Atlas
mut a := atlas.new(name: 'my_docs')!
// Scan a directory for collections
a.scan(path: '/path/to/docs')!
// Export to destination
a.export(destination: '/path/to/output')!
```
## Collections
Collections are directories marked with a `.collection` file.
### .collection File Format
```
name:my_collection
```
## Usage Examples
### Scanning for Collections
```v
mut a := atlas.new()!
a.scan(path: './docs')!
```
### Adding a Specific Collection
```v
a.add_collection(name: 'guides', path: './docs/guides')!
```
### Getting Pages
```v
// Get a page
page := a.page_get('guides:introduction')!
content := page.read_content()!
// Check if page exists
if a.page_exists('guides:setup') {
println('Setup guide found')
}
```
### Exporting
```v
// Export with Redis metadata
a.export(
destination: './output'
reset: true
redis: true
)!
```
## Redis Structure
When `redis: true` in export:
```
atlas:path -> hash of collection names to export paths
atlas:my_collection -> hash of file names to relative paths
```
## Key Differences from Doctree
- **No Processing**: Files are copied as-is
- **No Includes**: No `!!wiki.include` processing
- **No Definitions**: No `!!wiki.def` processing
- **No Link Resolution**: Markdown links are not modified
- **Simpler Structure**: Flat module organization
- **Faster**: No parsing overhead
## When to Use
Use **Atlas** when you need:
- Simple document organization
- Fast file copying without processing
- Basic metadata tracking
- Minimal overhead
Use **Doctree** when you need:
- Markdown processing and transformations
- Include/definition resolution
- Link rewriting
- Complex document workflows

103
lib/data/atlas/scan.v Normal file
View File

@@ -0,0 +1,103 @@
module atlas
import incubaid.herolib.core.pathlib
import incubaid.herolib.data.paramsparser
import incubaid.herolib.core.texttools
import os
@[params]
pub struct ScanArgs {
pub mut:
path string @[required]
}
// Scan a directory for collections
fn (mut a Atlas) scan_directory(mut dir pathlib.Path) ! {
if !dir.is_dir() {
return error('Path is not a directory: ${dir.path}')
}
// Check if this directory is a collection
if is_collection_dir(dir) {
collection_name := get_collection_name(mut dir)!
a.add_collection(path: dir.path, name: collection_name)!
return
}
// Scan subdirectories
entries := dir.list(recursive: false)!
for entry in entries.paths {
if !entry.is_dir() || should_skip_dir(entry) {
continue
}
mut mutable_entry := entry
a.scan_directory(mut mutable_entry)!
}
}
// Check if directory is a collection
fn is_collection_dir(path pathlib.Path) bool {
return path.file_exists('.collection')
}
// Get collection name from .collection file
fn get_collection_name(mut path pathlib.Path) !string {
mut collection_name := path.name()
mut filepath := path.file_get('.collection')!
content := filepath.read()!
if content.trim_space() != '' {
mut params := paramsparser.parse(content)!
if params.exists('name') {
collection_name = params.get('name')!
}
}
return texttools.name_fix(collection_name)
}
// Check if directory should be skipped
fn should_skip_dir(entry pathlib.Path) bool {
name := entry.name()
return name.starts_with('.') || name.starts_with('_')
}
// Scan collection directory for files
fn (mut c Collection) scan() ! {
c.scan_path(mut c.path)!
}
fn (mut c Collection) scan_path(mut dir pathlib.Path) ! {
entries := dir.list(recursive: false)!
for entry in entries.paths {
// Skip hidden files/dirs
if entry.name().starts_with('.') || entry.name().starts_with('_') {
continue
}
if entry.is_dir() {
// Recursively scan subdirectories
mut mutable_entry := entry
c.scan_path(mut mutable_entry)!
continue
}
// Process files based on extension
match entry.extension_lower() {
'md' {
mut mutable_entry := entry
c.add_page(mut mutable_entry)!
}
'png', 'jpg', 'jpeg', 'gif', 'svg' {
mut mutable_entry := entry
c.add_image(mut mutable_entry)!
}
else {
mut mutable_entry := entry
c.add_file(mut mutable_entry)!
}
}
}
}