...
This commit is contained in:
498
examples/data/atlas/atlas_loader.py
Normal file
498
examples/data/atlas/atlas_loader.py
Normal file
@@ -0,0 +1,498 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Atlas Collection Loader for Python
|
||||
|
||||
Load Atlas collections from .collection.json files created by the V Atlas module.
|
||||
This allows Python applications to access Atlas data without running V code.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class FileType(Enum):
|
||||
"""File type enumeration"""
|
||||
FILE = "file"
|
||||
IMAGE = "image"
|
||||
|
||||
|
||||
class CollectionErrorCategory(Enum):
|
||||
"""Error category enumeration matching V implementation"""
|
||||
CIRCULAR_INCLUDE = "circular_include"
|
||||
MISSING_INCLUDE = "missing_include"
|
||||
INCLUDE_SYNTAX_ERROR = "include_syntax_error"
|
||||
INVALID_PAGE_REFERENCE = "invalid_page_reference"
|
||||
FILE_NOT_FOUND = "file_not_found"
|
||||
INVALID_COLLECTION = "invalid_collection"
|
||||
GENERAL_ERROR = "general_error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CollectionError:
|
||||
"""Collection error matching V CollectionError struct"""
|
||||
category: str
|
||||
page_key: str = ""
|
||||
message: str = ""
|
||||
file: str = ""
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> 'CollectionError':
|
||||
"""Create from dictionary"""
|
||||
return cls(
|
||||
category=data.get('category', ''),
|
||||
page_key=data.get('page_key', ''),
|
||||
message=data.get('message', ''),
|
||||
file=data.get('file', '')
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""Human-readable error message"""
|
||||
location = ""
|
||||
if self.page_key:
|
||||
location = f" [{self.page_key}]"
|
||||
elif self.file:
|
||||
location = f" [{self.file}]"
|
||||
return f"[{self.category}]{location}: {self.message}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class File:
|
||||
"""File metadata matching V File struct"""
|
||||
name: str
|
||||
ext: str
|
||||
path: str
|
||||
ftype: str
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> 'File':
|
||||
"""Create from dictionary"""
|
||||
return cls(
|
||||
name=data['name'],
|
||||
ext=data['ext'],
|
||||
path=data['path'],
|
||||
ftype=data['ftype']
|
||||
)
|
||||
|
||||
@property
|
||||
def file_type(self) -> FileType:
|
||||
"""Get file type as enum"""
|
||||
return FileType(self.ftype)
|
||||
|
||||
@property
|
||||
def file_name(self) -> str:
|
||||
"""Get full filename with extension"""
|
||||
return f"{self.name}.{self.ext}"
|
||||
|
||||
def is_image(self) -> bool:
|
||||
"""Check if file is an image"""
|
||||
return self.file_type == FileType.IMAGE
|
||||
|
||||
def read(self) -> bytes:
|
||||
"""Read file content as bytes"""
|
||||
return Path(self.path).read_bytes()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Page:
|
||||
"""Page metadata matching V Page struct"""
|
||||
name: str
|
||||
path: str
|
||||
collection_name: str
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> 'Page':
|
||||
"""Create from dictionary"""
|
||||
return cls(
|
||||
name=data['name'],
|
||||
path=data['path'],
|
||||
collection_name=data['collection_name']
|
||||
)
|
||||
|
||||
def key(self) -> str:
|
||||
"""Get page key in format 'collection:page'"""
|
||||
return f"{self.collection_name}:{self.name}"
|
||||
|
||||
def read_content(self) -> str:
|
||||
"""Read page content from file"""
|
||||
return Path(self.path).read_text(encoding='utf-8')
|
||||
|
||||
|
||||
@dataclass
|
||||
class Collection:
|
||||
"""Collection matching V Collection struct"""
|
||||
name: str
|
||||
path: str
|
||||
pages: Dict[str, Page] = field(default_factory=dict)
|
||||
images: Dict[str, File] = field(default_factory=dict)
|
||||
files: Dict[str, File] = field(default_factory=dict)
|
||||
errors: List[CollectionError] = field(default_factory=list)
|
||||
|
||||
def page_get(self, name: str) -> Optional[Page]:
|
||||
"""Get a page by name"""
|
||||
return self.pages.get(name)
|
||||
|
||||
def page_exists(self, name: str) -> bool:
|
||||
"""Check if page exists"""
|
||||
return name in self.pages
|
||||
|
||||
def image_get(self, name: str) -> Optional[File]:
|
||||
"""Get an image by name"""
|
||||
return self.images.get(name)
|
||||
|
||||
def image_exists(self, name: str) -> bool:
|
||||
"""Check if image exists"""
|
||||
return name in self.images
|
||||
|
||||
def file_get(self, name: str) -> Optional[File]:
|
||||
"""Get a file by name"""
|
||||
return self.files.get(name)
|
||||
|
||||
def file_exists(self, name: str) -> bool:
|
||||
"""Check if file exists"""
|
||||
return name in self.files
|
||||
|
||||
def has_errors(self) -> bool:
|
||||
"""Check if collection has errors"""
|
||||
return len(self.errors) > 0
|
||||
|
||||
def error_summary(self) -> Dict[str, int]:
|
||||
"""Get error count by category"""
|
||||
summary = {}
|
||||
for err in self.errors:
|
||||
category = err.category
|
||||
summary[category] = summary.get(category, 0) + 1
|
||||
return summary
|
||||
|
||||
def print_errors(self):
|
||||
"""Print all errors to console"""
|
||||
if not self.has_errors():
|
||||
print(f"Collection {self.name}: No errors")
|
||||
return
|
||||
|
||||
print(f"\nCollection {self.name} - Errors ({len(self.errors)})")
|
||||
print("=" * 60)
|
||||
for err in self.errors:
|
||||
print(f" {err}")
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, json_path: Path) -> 'Collection':
|
||||
"""
|
||||
Load collection from .collection.json file
|
||||
|
||||
Args:
|
||||
json_path: Path to .collection.json file
|
||||
|
||||
Returns:
|
||||
Collection instance
|
||||
"""
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Parse pages - V outputs as map[string]Page
|
||||
pages = {}
|
||||
for name, page_data in data.get('pages', {}).items():
|
||||
pages[name] = Page.from_dict(page_data)
|
||||
|
||||
# Parse images - V outputs as map[string]File
|
||||
images = {}
|
||||
for name, file_data in data.get('images', {}).items():
|
||||
images[name] = File.from_dict(file_data)
|
||||
|
||||
# Parse files - V outputs as map[string]File
|
||||
files = {}
|
||||
for name, file_data in data.get('files', {}).items():
|
||||
files[name] = File.from_dict(file_data)
|
||||
|
||||
# Parse errors - V outputs as []CollectionError
|
||||
errors = []
|
||||
for err_data in data.get('errors', []):
|
||||
errors.append(CollectionError.from_dict(err_data))
|
||||
|
||||
return cls(
|
||||
name=data['name'],
|
||||
path=data['path'],
|
||||
pages=pages,
|
||||
images=images,
|
||||
files=files,
|
||||
errors=errors
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Atlas:
|
||||
"""Atlas matching V Atlas struct"""
|
||||
name: str = "default"
|
||||
collections: Dict[str, Collection] = field(default_factory=dict)
|
||||
|
||||
def add_collection(self, collection: Collection):
|
||||
"""Add a collection to the atlas"""
|
||||
self.collections[collection.name] = collection
|
||||
|
||||
def get_collection(self, name: str) -> Optional[Collection]:
|
||||
"""Get a collection by name"""
|
||||
return self.collections.get(name)
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if collection exists"""
|
||||
return name in self.collections
|
||||
|
||||
def page_get(self, key: str) -> Optional[Page]:
|
||||
"""
|
||||
Get a page using format 'collection:page'
|
||||
|
||||
Args:
|
||||
key: Page key in format 'collection:page'
|
||||
|
||||
Returns:
|
||||
Page or None if not found
|
||||
"""
|
||||
parts = key.split(':', 1)
|
||||
if len(parts) != 2:
|
||||
return None
|
||||
|
||||
col = self.get_collection(parts[0])
|
||||
if not col:
|
||||
return None
|
||||
|
||||
return col.page_get(parts[1])
|
||||
|
||||
def page_exists(self, key: str) -> bool:
|
||||
"""Check if page exists using format 'collection:page'"""
|
||||
return self.page_get(key) is not None
|
||||
|
||||
def image_get(self, key: str) -> Optional[File]:
|
||||
"""Get an image using format 'collection:image'"""
|
||||
parts = key.split(':', 1)
|
||||
if len(parts) != 2:
|
||||
return None
|
||||
|
||||
col = self.get_collection(parts[0])
|
||||
if not col:
|
||||
return None
|
||||
|
||||
return col.image_get(parts[1])
|
||||
|
||||
def image_exists(self, key: str) -> bool:
|
||||
"""Check if image exists using format 'collection:image'"""
|
||||
return self.image_get(key) is not None
|
||||
|
||||
def file_get(self, key: str) -> Optional[File]:
|
||||
"""Get a file using format 'collection:file'"""
|
||||
parts = key.split(':', 1)
|
||||
if len(parts) != 2:
|
||||
return None
|
||||
|
||||
col = self.get_collection(parts[0])
|
||||
if not col:
|
||||
return None
|
||||
|
||||
return col.file_get(parts[1])
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collection names"""
|
||||
return sorted(self.collections.keys())
|
||||
|
||||
def list_pages(self) -> Dict[str, List[str]]:
|
||||
"""List all pages grouped by collection"""
|
||||
result = {}
|
||||
for col_name, col in self.collections.items():
|
||||
result[col_name] = sorted(col.pages.keys())
|
||||
return result
|
||||
|
||||
def has_errors(self) -> bool:
|
||||
"""Check if any collection has errors"""
|
||||
return any(col.has_errors() for col in self.collections.values())
|
||||
|
||||
def print_all_errors(self):
|
||||
"""Print errors from all collections"""
|
||||
for col in self.collections.values():
|
||||
if col.has_errors():
|
||||
col.print_errors()
|
||||
|
||||
@classmethod
|
||||
def load_collection(cls, path: str, name: str = "default") -> 'Atlas':
|
||||
"""
|
||||
Load a single collection from a path.
|
||||
|
||||
Args:
|
||||
path: Path to the collection directory containing .collection.json
|
||||
name: Name for the atlas instance
|
||||
|
||||
Returns:
|
||||
Atlas with the loaded collection
|
||||
|
||||
Example:
|
||||
atlas = Atlas.load_collection('/path/to/my_collection')
|
||||
col = atlas.get_collection('my_collection')
|
||||
"""
|
||||
atlas = cls(name=name)
|
||||
collection_path = Path(path) / '.collection.json'
|
||||
|
||||
if not collection_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"No .collection.json found at {path}\n"
|
||||
f"Make sure to run collection.save() in V first"
|
||||
)
|
||||
|
||||
collection = Collection.from_json(collection_path)
|
||||
atlas.add_collection(collection)
|
||||
|
||||
return atlas
|
||||
|
||||
@classmethod
|
||||
def load_from_directory(cls, path: str, name: str = "default") -> 'Atlas':
|
||||
"""
|
||||
Walk directory tree and load all collections.
|
||||
|
||||
Args:
|
||||
path: Root path to scan for .collection.json files
|
||||
name: Name for the atlas instance
|
||||
|
||||
Returns:
|
||||
Atlas with all found collections
|
||||
|
||||
Example:
|
||||
atlas = Atlas.load_from_directory('/path/to/docs')
|
||||
print(f"Loaded {len(atlas.collections)} collections")
|
||||
"""
|
||||
atlas = cls(name=name)
|
||||
root = Path(path)
|
||||
|
||||
if not root.exists():
|
||||
raise FileNotFoundError(f"Path not found: {path}")
|
||||
|
||||
# Walk directory tree looking for .collection.json files
|
||||
for json_file in root.rglob('.collection.json'):
|
||||
try:
|
||||
collection = Collection.from_json(json_file)
|
||||
atlas.add_collection(collection)
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to load {json_file}: {e}")
|
||||
|
||||
if len(atlas.collections) == 0:
|
||||
print(f"Warning: No collections found in {path}")
|
||||
|
||||
return atlas
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Example Usage Functions
|
||||
# ============================================================================
|
||||
|
||||
def example_load_single_collection():
|
||||
"""Example: Load a single collection"""
|
||||
print("\n" + "="*60)
|
||||
print("Example 1: Load Single Collection")
|
||||
print("="*60)
|
||||
|
||||
atlas = Atlas.load_collection(
|
||||
path='/tmp/atlas_test/col1',
|
||||
name='my_atlas'
|
||||
)
|
||||
|
||||
# Get collection
|
||||
col = atlas.get_collection('col1')
|
||||
if col:
|
||||
print(f"\nLoaded collection: {col.name}")
|
||||
print(f" Path: {col.path}")
|
||||
print(f" Pages: {len(col.pages)}")
|
||||
print(f" Images: {len(col.images)}")
|
||||
print(f" Files: {len(col.files)}")
|
||||
|
||||
# Print errors if any
|
||||
if col.has_errors():
|
||||
col.print_errors()
|
||||
|
||||
|
||||
def example_load_all_collections():
|
||||
"""Example: Load all collections from a directory tree"""
|
||||
print("\n" + "="*60)
|
||||
print("Example 2: Load All Collections")
|
||||
print("="*60)
|
||||
|
||||
atlas = Atlas.load_from_directory(
|
||||
path='/tmp/atlas_test',
|
||||
name='docs_atlas'
|
||||
)
|
||||
|
||||
print(f"\nLoaded {len(atlas.collections)} collections:")
|
||||
|
||||
# List all collections
|
||||
for col_name in atlas.list_collections():
|
||||
col = atlas.get_collection(col_name)
|
||||
print(f"\n Collection: {col_name}")
|
||||
print(f" Path: {col.path}")
|
||||
print(f" Pages: {len(col.pages)}")
|
||||
print(f" Images: {len(col.images)}")
|
||||
print(f" Errors: {len(col.errors)}")
|
||||
|
||||
|
||||
def example_access_pages():
|
||||
"""Example: Access pages and content"""
|
||||
print("\n" + "="*60)
|
||||
print("Example 3: Access Pages and Content")
|
||||
print("="*60)
|
||||
|
||||
atlas = Atlas.load_from_directory('/tmp/atlas_test')
|
||||
|
||||
# Get a specific page
|
||||
page = atlas.page_get('col1:page1')
|
||||
if page:
|
||||
print(f"\nPage: {page.name}")
|
||||
print(f" Key: {page.key()}")
|
||||
print(f" Path: {page.path}")
|
||||
|
||||
# Read content
|
||||
content = page.read_content()
|
||||
print(f" Content length: {len(content)} chars")
|
||||
print(f" First 100 chars: {content[:100]}")
|
||||
|
||||
# List all pages
|
||||
print("\nAll pages:")
|
||||
pages = atlas.list_pages()
|
||||
for col_name, page_names in pages.items():
|
||||
print(f"\n {col_name}:")
|
||||
for page_name in page_names:
|
||||
print(f" - {page_name}")
|
||||
|
||||
|
||||
def example_error_handling():
|
||||
"""Example: Working with errors"""
|
||||
print("\n" + "="*60)
|
||||
print("Example 4: Error Handling")
|
||||
print("="*60)
|
||||
|
||||
atlas = Atlas.load_from_directory('/tmp/atlas_test')
|
||||
|
||||
# Check for errors across all collections
|
||||
if atlas.has_errors():
|
||||
print("\nFound errors in collections:")
|
||||
atlas.print_all_errors()
|
||||
else:
|
||||
print("\nNo errors found!")
|
||||
|
||||
# Get error summary for a specific collection
|
||||
col = atlas.get_collection('col1')
|
||||
if col and col.has_errors():
|
||||
summary = col.error_summary()
|
||||
print(f"\nError summary for {col.name}:")
|
||||
for category, count in summary.items():
|
||||
print(f" {category}: {count}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("Atlas Loader - Python Implementation")
|
||||
print("="*60)
|
||||
print("\nThis script demonstrates loading Atlas collections")
|
||||
print("from .collection.json files created by the V Atlas module.")
|
||||
|
||||
# Uncomment to run examples:
|
||||
# example_load_single_collection()
|
||||
# example_load_all_collections()
|
||||
# example_access_pages()
|
||||
# example_error_handling()
|
||||
|
||||
print("\nUncomment example functions in __main__ to see them in action.")
|
||||
83
examples/data/atlas/example_save_load.vsh
Executable file
83
examples/data/atlas/example_save_load.vsh
Executable file
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env -S v -n -w -cg -gc none -cc tcc -d use_openssl -enable-globals run
|
||||
|
||||
import incubaid.herolib.data.atlas
|
||||
import incubaid.herolib.core.pathlib
|
||||
import os
|
||||
|
||||
// Example: Save and Load Atlas Collections
|
||||
|
||||
println('Atlas Save/Load Example')
|
||||
println('============================================================')
|
||||
|
||||
// Setup test directory
|
||||
test_dir := '/tmp/atlas_example'
|
||||
os.rmdir_all(test_dir) or {}
|
||||
os.mkdir_all(test_dir)!
|
||||
|
||||
// Create a collection with some content
|
||||
col_path := '${test_dir}/docs'
|
||||
os.mkdir_all(col_path)!
|
||||
|
||||
mut cfile := pathlib.get_file(path: '${col_path}/.collection', create: true)!
|
||||
cfile.write('name:docs')!
|
||||
|
||||
mut page1 := pathlib.get_file(path: '${col_path}/intro.md', create: true)!
|
||||
page1.write('# Introduction\n\nWelcome to the docs!')!
|
||||
|
||||
mut page2 := pathlib.get_file(path: '${col_path}/guide.md', create: true)!
|
||||
page2.write('# Guide\n\n!!include docs:intro\n\nMore content here.')!
|
||||
|
||||
// Create and scan atlas
|
||||
println('\n1. Creating Atlas and scanning...')
|
||||
mut a := atlas.new(name: 'my_docs')!
|
||||
a.scan(path: test_dir)!
|
||||
|
||||
println(' Found ${a.collections.len} collection(s)')
|
||||
|
||||
// Validate links
|
||||
println('\n2. Validating links...')
|
||||
a.validate_links()!
|
||||
|
||||
col := a.get_collection('docs')!
|
||||
if col.has_errors() {
|
||||
println(' Errors found:')
|
||||
col.print_errors()
|
||||
} else {
|
||||
println(' No errors found!')
|
||||
}
|
||||
|
||||
// Save all collections
|
||||
println('\n3. Saving collections to .collection.json...')
|
||||
a.save_all()!
|
||||
println(' Saved to ${col_path}/.collection.json')
|
||||
|
||||
// Load in a new atlas
|
||||
println('\n4. Loading collections in new Atlas...')
|
||||
mut a2 := atlas.new(name: 'loaded_docs')!
|
||||
a2.load_from_directory(test_dir)!
|
||||
|
||||
println(' Loaded ${a2.collections.len} collection(s)')
|
||||
|
||||
// Access loaded data
|
||||
println('\n5. Accessing loaded data...')
|
||||
loaded_col := a2.get_collection('docs')!
|
||||
println(' Collection: ${loaded_col.name}')
|
||||
println(' Pages: ${loaded_col.pages.len}')
|
||||
|
||||
for name, page in loaded_col.pages {
|
||||
println(' - ${name}: ${page.path.path}')
|
||||
}
|
||||
|
||||
// Read page content
|
||||
println('\n6. Reading page content...')
|
||||
mut intro_page := loaded_col.page_get('intro')!
|
||||
content := intro_page.read_content()!
|
||||
println(' intro.md content:')
|
||||
println(' ${content}')
|
||||
|
||||
println('\n✓ Example completed successfully!')
|
||||
println('\nNow you can use the Python loader:')
|
||||
println(' python3 lib/data/atlas/atlas_loader.py')
|
||||
|
||||
// Cleanup
|
||||
os.rmdir_all(test_dir) or {}
|
||||
@@ -341,4 +341,108 @@ fn test_cross_collection_links() {
|
||||
|
||||
fixed := page1.read()!
|
||||
assert fixed.contains('[Link to col2](col2:page2)') // Unchanged
|
||||
fn test_save_and_load() {
|
||||
// Setup
|
||||
col_path := '${test_base}/save_test'
|
||||
os.mkdir_all(col_path)!
|
||||
|
||||
mut cfile := pathlib.get_file(path: '${col_path}/.collection', create: true)!
|
||||
cfile.write('name:test_col')!
|
||||
|
||||
mut page := pathlib.get_file(path: '${col_path}/page1.md', create: true)!
|
||||
page.write('# Page 1\n\nContent here.')!
|
||||
|
||||
// Create and save
|
||||
mut a := new(name: 'test')!
|
||||
a.add_collection(name: 'test_col', path: col_path)!
|
||||
a.save_all()!
|
||||
|
||||
assert os.exists('${col_path}/.collection.json')
|
||||
|
||||
// Load in new atlas
|
||||
mut a2 := new(name: 'loaded')!
|
||||
a2.load_collection(col_path)!
|
||||
|
||||
assert a2.collections.len == 1
|
||||
col := a2.get_collection('test_col')!
|
||||
assert col.pages.len == 1
|
||||
assert col.page_exists('page1')
|
||||
|
||||
// Verify page can read content
|
||||
mut page_loaded := col.page_get('page1')!
|
||||
content := page_loaded.read_content()!
|
||||
assert content.contains('# Page 1')
|
||||
}
|
||||
|
||||
fn test_save_with_errors() {
|
||||
col_path := '${test_base}/error_save_test'
|
||||
os.mkdir_all(col_path)!
|
||||
|
||||
mut cfile := pathlib.get_file(path: '${col_path}/.collection', create: true)!
|
||||
cfile.write('name:err_col')!
|
||||
|
||||
mut a := new(name: 'test')!
|
||||
mut col := a.new_collection(name: 'err_col', path: col_path)!
|
||||
|
||||
// Add some errors
|
||||
col.error(
|
||||
category: .missing_include
|
||||
page_key: 'err_col:page1'
|
||||
message: 'Test error 1'
|
||||
)
|
||||
|
||||
col.error(
|
||||
category: .invalid_page_reference
|
||||
page_key: 'err_col:page2'
|
||||
message: 'Test error 2'
|
||||
)
|
||||
|
||||
a.collections['err_col'] = &col
|
||||
|
||||
// Save
|
||||
col.save()!
|
||||
|
||||
// Load
|
||||
mut a2 := new(name: 'loaded')!
|
||||
loaded_col := a2.load_collection(col_path)!
|
||||
|
||||
// Verify errors persisted
|
||||
assert loaded_col.errors.len == 2
|
||||
assert loaded_col.error_cache.len == 2
|
||||
}
|
||||
|
||||
fn test_load_from_directory() {
|
||||
// Setup multiple collections
|
||||
col1_path := '${test_base}/load_dir/col1'
|
||||
col2_path := '${test_base}/load_dir/col2'
|
||||
|
||||
os.mkdir_all(col1_path)!
|
||||
os.mkdir_all(col2_path)!
|
||||
|
||||
mut cfile1 := pathlib.get_file(path: '${col1_path}/.collection', create: true)!
|
||||
cfile1.write('name:col1')!
|
||||
|
||||
mut cfile2 := pathlib.get_file(path: '${col2_path}/.collection', create: true)!
|
||||
cfile2.write('name:col2')!
|
||||
|
||||
mut page1 := pathlib.get_file(path: '${col1_path}/page1.md', create: true)!
|
||||
page1.write('# Page 1')!
|
||||
|
||||
mut page2 := pathlib.get_file(path: '${col2_path}/page2.md', create: true)!
|
||||
page2.write('# Page 2')!
|
||||
|
||||
// Create and save
|
||||
mut a := new(name: 'test')!
|
||||
a.add_collection(name: 'col1', path: col1_path)!
|
||||
a.add_collection(name: 'col2', path: col2_path)!
|
||||
a.save_all()!
|
||||
|
||||
// Load from directory
|
||||
mut a2 := new(name: 'loaded')!
|
||||
a2.load_from_directory('${test_base}/load_dir')!
|
||||
|
||||
assert a2.collections.len == 2
|
||||
assert a2.get_collection('col1')!.page_exists('page1')
|
||||
assert a2.get_collection('col2')!.page_exists('page2')
|
||||
}
|
||||
}
|
||||
@@ -377,14 +377,256 @@ img_path := redis.hget('atlas:guides', 'logo.png')!
|
||||
println('Logo image: ${img_path}') // Output: img/logo.png
|
||||
```
|
||||
|
||||
### Disabling Redis
|
||||
|
||||
If you don't need Redis metadata storage:
|
||||
## Atlas Save/Load Functionality
|
||||
|
||||
This document describes the save/load functionality for Atlas collections, which allows you to persist collection metadata to JSON files and load them in both V and Python.
|
||||
|
||||
## Overview
|
||||
|
||||
The Atlas module now supports:
|
||||
- **Saving collections** to `.collection.json` files
|
||||
- **Loading collections** from `.collection.json` files in V
|
||||
- **Loading collections** from `.collection.json` files in Python
|
||||
|
||||
This enables:
|
||||
1. Persistence of collection metadata (pages, images, files, errors)
|
||||
2. Cross-language access to Atlas data
|
||||
3. Faster loading without re-scanning directories
|
||||
|
||||
## V Implementation
|
||||
|
||||
### Saving Collections
|
||||
|
||||
```v
|
||||
a.export(
|
||||
destination: './output'
|
||||
redis: false // Skip Redis storage
|
||||
)!
|
||||
import incubaid.herolib.data.atlas
|
||||
|
||||
// Create and scan atlas
|
||||
mut a := atlas.new(name: 'my_docs')!
|
||||
a.scan(path: './docs')!
|
||||
|
||||
// Save all collections (creates .collection.json in each collection dir)
|
||||
a.save_all()!
|
||||
|
||||
// Or save a single collection
|
||||
col := a.get_collection('guides')!
|
||||
col.save()!
|
||||
```
|
||||
|
||||
### Loading Collections
|
||||
|
||||
```v
|
||||
import incubaid.herolib.data.atlas
|
||||
|
||||
// Load single collection
|
||||
mut a := atlas.new(name: 'loaded')!
|
||||
mut col := a.load_collection('/path/to/collection')!
|
||||
|
||||
println('Pages: ${col.pages.len}')
|
||||
|
||||
// Load all collections from directory tree
|
||||
mut a2 := atlas.new(name: 'all_docs')!
|
||||
a2.load_from_directory('./docs')!
|
||||
|
||||
println('Loaded ${a2.collections.len} collections')
|
||||
```
|
||||
|
||||
### What Gets Saved
|
||||
|
||||
The `.collection.json` file contains:
|
||||
- Collection name and path
|
||||
- All pages (name, path, collection_name)
|
||||
- All images (name, ext, path, ftype)
|
||||
- All files (name, ext, path, ftype)
|
||||
- All errors (category, page_key, message, file)
|
||||
|
||||
**Note:** Circular references (`atlas` and `collection` pointers) are automatically skipped using the `[skip]` attribute and reconstructed during load.
|
||||
|
||||
## Python Implementation
|
||||
|
||||
### Installation
|
||||
|
||||
The Python loader is a standalone script with no external dependencies (uses only Python stdlib):
|
||||
|
||||
```bash
|
||||
# No installation needed - just use the script
|
||||
python3 lib/data/atlas/atlas_loader.py
|
||||
```
|
||||
|
||||
### Loading Collections
|
||||
|
||||
```python
|
||||
from atlas_loader import Atlas
|
||||
|
||||
# Load single collection
|
||||
atlas = Atlas.load_collection('/path/to/collection')
|
||||
|
||||
# Or load all collections from directory tree
|
||||
atlas = Atlas.load_from_directory('/path/to/docs')
|
||||
|
||||
# Access collections
|
||||
col = atlas.get_collection('guides')
|
||||
print(f"Pages: {len(col.pages)}")
|
||||
|
||||
# Access pages
|
||||
page = atlas.page_get('guides:intro')
|
||||
if page:
|
||||
content = page.read_content()
|
||||
print(content)
|
||||
|
||||
# Check for errors
|
||||
if atlas.has_errors():
|
||||
atlas.print_all_errors()
|
||||
```
|
||||
|
||||
### Python API
|
||||
|
||||
#### Atlas Class
|
||||
|
||||
- `Atlas.load_collection(path, name='default')` - Load single collection
|
||||
- `Atlas.load_from_directory(path, name='default')` - Load all collections from directory tree
|
||||
- `atlas.get_collection(name)` - Get collection by name
|
||||
- `atlas.page_get(key)` - Get page using 'collection:page' format
|
||||
- `atlas.image_get(key)` - Get image using 'collection:image' format
|
||||
- `atlas.file_get(key)` - Get file using 'collection:file' format
|
||||
- `atlas.list_collections()` - List all collection names
|
||||
- `atlas.list_pages()` - List all pages grouped by collection
|
||||
- `atlas.has_errors()` - Check if any collection has errors
|
||||
- `atlas.print_all_errors()` - Print errors from all collections
|
||||
|
||||
#### Collection Class
|
||||
|
||||
- `collection.page_get(name)` - Get page by name
|
||||
- `collection.image_get(name)` - Get image by name
|
||||
- `collection.file_get(name)` - Get file by name
|
||||
- `collection.has_errors()` - Check if collection has errors
|
||||
- `collection.error_summary()` - Get error count by category
|
||||
- `collection.print_errors()` - Print all errors
|
||||
|
||||
#### Page Class
|
||||
|
||||
- `page.key()` - Get page key in format 'collection:page'
|
||||
- `page.read_content()` - Read page content from file
|
||||
|
||||
#### File Class
|
||||
|
||||
- `file.file_name` - Get full filename with extension
|
||||
- `file.is_image()` - Check if file is an image
|
||||
- `file.read()` - Read file content as bytes
|
||||
|
||||
## Workflow
|
||||
|
||||
### 1. V: Create and Save
|
||||
|
||||
```v
|
||||
#!/usr/bin/env -S v -n -w -cg -gc none -cc tcc -d use_openssl -enable-globals run
|
||||
|
||||
import incubaid.herolib.data.atlas
|
||||
|
||||
// Create atlas and scan
|
||||
mut a := atlas.new(name: 'my_docs')!
|
||||
a.scan(path: './docs')!
|
||||
|
||||
// Validate
|
||||
a.validate_links()!
|
||||
|
||||
// Save all collections (creates .collection.json in each collection dir)
|
||||
a.save_all()!
|
||||
|
||||
println('Saved ${a.collections.len} collections')
|
||||
```
|
||||
|
||||
### 2. V: Load and Use
|
||||
|
||||
```v
|
||||
#!/usr/bin/env -S v -n -w -cg -gc none -cc tcc -d use_openssl -enable-globals run
|
||||
|
||||
import incubaid.herolib.data.atlas
|
||||
|
||||
// Load single collection
|
||||
mut a := atlas.new(name: 'loaded')!
|
||||
mut col := a.load_collection('/path/to/collection')!
|
||||
|
||||
println('Pages: ${col.pages.len}')
|
||||
|
||||
// Load all from directory
|
||||
mut a2 := atlas.new(name: 'all_docs')!
|
||||
a2.load_from_directory('./docs')!
|
||||
|
||||
println('Loaded ${a2.collections.len} collections')
|
||||
```
|
||||
|
||||
### 3. Python: Load and Use
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from atlas_loader import Atlas
|
||||
|
||||
# Load single collection
|
||||
atlas = Atlas.load_collection('/path/to/collection')
|
||||
|
||||
# Or load all collections
|
||||
atlas = Atlas.load_from_directory('/path/to/docs')
|
||||
|
||||
# Access pages
|
||||
page = atlas.page_get('guides:intro')
|
||||
if page:
|
||||
content = page.read_content()
|
||||
print(content)
|
||||
|
||||
# Check errors
|
||||
if atlas.has_errors():
|
||||
atlas.print_all_errors()
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
After saving, each collection directory will contain:
|
||||
|
||||
```
|
||||
collection_dir/
|
||||
├── .collection # Original collection config
|
||||
├── .collection.json # Saved collection metadata (NEW)
|
||||
├── page1.md
|
||||
├── page2.md
|
||||
└── img/
|
||||
└── image1.png
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
Errors are preserved during save/load:
|
||||
|
||||
```v
|
||||
// V: Errors are saved
|
||||
mut a := atlas.new()!
|
||||
a.scan(path: './docs')!
|
||||
a.validate_links()! // May generate errors
|
||||
a.save_all()! // Errors are saved to .collection.json
|
||||
|
||||
// V: Errors are loaded
|
||||
mut a2 := atlas.new()!
|
||||
a2.load_from_directory('./docs')!
|
||||
col := a2.get_collection('guides')!
|
||||
if col.has_errors() {
|
||||
col.print_errors()
|
||||
}
|
||||
```
|
||||
|
||||
```python
|
||||
# Python: Access errors
|
||||
atlas = Atlas.load_from_directory('./docs')
|
||||
|
||||
if atlas.has_errors():
|
||||
atlas.print_all_errors()
|
||||
|
||||
# Get error summary
|
||||
col = atlas.get_collection('guides')
|
||||
if col.has_errors():
|
||||
summary = col.error_summary()
|
||||
for category, count in summary.items():
|
||||
print(f"{category}: {count}")
|
||||
```
|
||||
|
||||
|
||||
|
||||
76
lib/data/atlas/save.v
Normal file
76
lib/data/atlas/save.v
Normal file
@@ -0,0 +1,76 @@
|
||||
module atlas
|
||||
|
||||
import json
|
||||
import incubaid.herolib.core.pathlib
|
||||
|
||||
// Save collection to .collection.json in the collection directory
|
||||
pub fn (c Collection) save() ! {
|
||||
// json.encode automatically skips fields marked with [skip]
|
||||
json_str := json.encode(c)
|
||||
|
||||
mut json_file := pathlib.get_file(
|
||||
path: '${c.path.path}/.collection.json'
|
||||
create: true
|
||||
)!
|
||||
|
||||
json_file.write(json_str)!
|
||||
}
|
||||
|
||||
// Save all collections in atlas to their respective directories
|
||||
pub fn (a Atlas) save_all() ! {
|
||||
for _, col in a.collections {
|
||||
col.save()!
|
||||
}
|
||||
}
|
||||
|
||||
// Load collection from .collection.json file
|
||||
pub fn (mut a Atlas) load_collection(path string) !&Collection {
|
||||
mut json_file := pathlib.get_file(path: '${path}/.collection.json')!
|
||||
json_str := json_file.read()!
|
||||
|
||||
mut col := json.decode(Collection, json_str)!
|
||||
|
||||
// Fix circular references that were skipped during encode
|
||||
col.atlas = &a
|
||||
|
||||
// Rebuild error cache from errors
|
||||
col.error_cache = map[string]bool{}
|
||||
for err in col.errors {
|
||||
col.error_cache[err.hash()] = true
|
||||
}
|
||||
|
||||
// Fix page references to collection
|
||||
for name, mut page in col.pages {
|
||||
page.collection = &col
|
||||
col.pages[name] = page
|
||||
}
|
||||
|
||||
a.collections[col.name] = &col
|
||||
return &col
|
||||
}
|
||||
|
||||
// Load all collections from a directory tree
|
||||
pub fn (mut a Atlas) load_from_directory(path string) ! {
|
||||
mut dir := pathlib.get_dir(path: path)!
|
||||
a.scan_and_load(mut dir)!
|
||||
}
|
||||
|
||||
// Scan directory for .collection.json files and load them
|
||||
fn (mut a Atlas) scan_and_load(mut dir pathlib.Path) ! {
|
||||
// Check if this directory has .collection.json
|
||||
if dir.file_exists('.collection.json') {
|
||||
a.load_collection(dir.path)!
|
||||
return
|
||||
}
|
||||
|
||||
// Scan subdirectories
|
||||
mut entries := dir.list(recursive: false)!
|
||||
for mut entry in entries.paths {
|
||||
if !entry.is_dir() || should_skip_dir(entry) {
|
||||
continue
|
||||
}
|
||||
|
||||
mut mutable_entry := entry
|
||||
a.scan_and_load(mut mutable_entry)!
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user