MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

Merge branch 'development' into development_nile_installers

Browse Source 
* development: (27 commits)
  ...
  ...
  fix: Ignore regex_convert_test.v test
  refactor: Replace codewalker with pathlib and filemap
  ...
  ...
  ...
  ...
  ...
  ...
  ...
  ...
  ...
  ...
  codewalker
  fix: Iterate over product requirement documents directly
  ...
  ...
  ...
  ...
  ...
This commit is contained in:
despiegk
2025-11-25 18:38:53 +01:00
parent 7e9bc1c41e a96d6e8aaa
commit 253e26aec6
94 changed files with 5892 additions and 1558 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/ai/client/aiclient.v
View File

@@ -1,7 +1,5 @@
module client
import incubaid.herolib.core.pathlib
@[heap]
pub struct AIClient {
pub mut:

52
lib/ai/client/aiclient_llm.v
View File

@@ -5,14 +5,16 @@ import os
pub struct AIClientLLMs {
pub mut:
llm_maverick &openai.OpenAI
llm_qwen &openai.OpenAI
llm_120b &openai.OpenAI
llm_best &openai.OpenAI
llm_flash &openai.OpenAI
llm_pro &openai.OpenAI
llm_morph &openai.OpenAI
llm_embed &openai.OpenAI
llm_maverick &openai.OpenAI
llm_qwen &openai.OpenAI
llm_120b &openai.OpenAI
llm_best &openai.OpenAI
llm_flash &openai.OpenAI
llm_pro &openai.OpenAI
llm_morph &openai.OpenAI
llm_embed &openai.OpenAI
llm_local &openai.OpenAI
llm_embed_local &openai.OpenAI
}
// Initialize all LLM clients
@@ -71,7 +73,7 @@ pub fn llms_init() !AIClientLLMs {
name: 'pro'
api_key: openrouter_key
url: 'https://api.openrouter.ai/api/v1'
model_default: 'google/gemini-2.5-pro'
model_default: 'google/gemini-3.0-pro'
}
openai.set(pro_client)!
@@ -91,14 +93,30 @@ pub fn llms_init() !AIClientLLMs {
}
openai.set(embed_client)!
mut local_client := openai.OpenAI{
name: 'local'
url: 'http://localhost:1234/v1'
model_default: 'google/gemma-3-12b'
}
openai.set(local_client)!
mut local_embed_client := openai.OpenAI{
name: 'embedlocal'
url: 'http://localhost:1234/v1'
model_default: 'text-embedding-nomic-embed-text-v1.5:2'
}
openai.set(local_embed_client)!
return AIClientLLMs{
llm_maverick: openai.get(name: 'maverick')!
llm_qwen: openai.get(name: 'qwen')!
llm_120b: openai.get(name: 'llm_120b')!
llm_best: openai.get(name: 'best')!
llm_flash: openai.get(name: 'flash')!
llm_pro: openai.get(name: 'pro')!
llm_morph: openai.get(name: 'morph')!
llm_embed: openai.get(name: 'embed')!
llm_maverick: openai.get(name: 'maverick')!
llm_qwen: openai.get(name: 'qwen')!
llm_120b: openai.get(name: 'llm_120b')!
llm_best: openai.get(name: 'best')!
llm_flash: openai.get(name: 'flash')!
llm_pro: openai.get(name: 'pro')!
llm_morph: openai.get(name: 'morph')!
llm_embed: openai.get(name: 'embed')!
llm_local: openai.get(name: 'local')!
llm_embed_local: openai.get(name: 'embedlocal')!
}
}

2
lib/ai/client/aiclient_models.v
View File

@@ -9,6 +9,7 @@ pub enum LLMEnum {
flash
pro
morph
local
}
fn llm_to_model_url(model LLMEnum) !(string, string) {
@@ -22,5 +23,6 @@ fn llm_to_model_url(model LLMEnum) !(string, string) {
.flash { 'google/gemini-2.5-flash', 'https://api.openrouter.ai/api/v1' }
.pro { 'google/gemini-2.5-pro', 'https://api.openrouter.ai/api/v1' }
.morph { 'morph/morph-v3-fast', 'https://api.openrouter.ai/api/v1' }
.local { 'google/gemma-3-12b', 'http://localhost:1234/v1' }
}
}

5
lib/ai/client/aiclient_validate.v
View File

@@ -10,11 +10,10 @@ pub fn validate_vlang_content(path pathlib.Path) !string {
// Use `v fmt -check` to validate V language syntax
// If there are any formatting issues, `v fmt -check` will return a non-zero exit code
// and print the issues to stderr.
res := os.system('v fmt -check ${path.str()}')
res := os.system('v fmt -check "${path.str()}" 2>/dev/null')
if res != 0 {
return 'V language syntax validation failed. Please check the file for errors.'
return 'V language syntax validation failed. File has formatting or syntax errors.'
}
// TODO: do 'v filepath' d and check if errors return, if no, then remove the compiled binary if its there, if it goes wrong do same
return '' // empty means no error
}

159
lib/ai/client/aiclient_write.v
View File

@@ -5,72 +5,173 @@ import incubaid.herolib.ui.console
import incubaid.herolib.clients.openai
import os
// TODO: do as params for the function
// WritePromptArgs holds the parameters for write_from_prompt function
@[params]
pub struct WritePromptArgs {
pub mut:
path pathlib.Path
prompt string
models []LLMEnum = [.best]
temperature f64 = 0.5
max_tokens int = 16000
system_prompt string = 'You are a helpful assistant that modifies files based on user instructions.'
}
pub fn (mut ac AIClient) write_from_prompt(path_ pathlib.Path, prompt string, models []LLMEnum) ! {
mut mypath := path_
// write_from_prompt modifies a file based on AI-generated modification instructions
//
// The process:
// 1. Uses the first model to generate modification instructions from the prompt
// 2. Uses the morph model to apply those instructions to the original content
// 3. Validates the result based on file type (.v, .md, .yaml, .json)
// 4. On validation failure, retries with the next model in the list
// 5. Restores from backup if all models fail
pub fn (mut ac AIClient) write_from_prompt(args WritePromptArgs) ! {
mut mypath := args.path
original_content := mypath.read()!
mut backup_path := pathlib.get_file(path: '${mypath.path}.backup', create: true)!
backup_path.write(original_content)!
mut selected_models := models.clone()
mut selected_models := args.models.clone()
if selected_models.len == 0 {
selected_models = [.best] // Default to best model if none provided
selected_models = [.best]
}
for model_enum in selected_models {
model_name, base_url := llm_to_model_url(model_enum)!
mut llm_client := openai.get(name: model_enum.str())! // Assuming model_enum.str() matches the name used in llms_init
model_name, _ := llm_to_model_url(model_enum)!
// 3. Use first model (or default best) to process prompt
// This part needs to be implemented based on how the OpenAI client's chat completion works
// For now, let's assume a simple completion call
// This is a placeholder and needs actual implementation based on the OpenAI client's chat completion method
// For example:
// completion := llm_client.chat_completion(prompt)!
// instructions := completion.choices[0].message.content
// Step 1: Get modification instructions from the selected model
// Get the appropriate LLM client for instruction generation
mut llm_client := get_llm_client(mut ac, model_enum)
// For now, let's just use the prompt as the "instructions" for modification
instructions := prompt
instruction_prompt := generate_instruction_prompt(original_content, mypath.ext()!,
args.prompt)
// 5. Use morph model to merge original + instructions
// This is a placeholder for the merging logic
// For now, let's just replace the content with instructions
new_content := instructions // This needs to be replaced with actual merging logic
instructions_response := llm_client.chat_completion(
message: instruction_prompt
temperature: args.temperature
max_completion_tokens: args.max_tokens
)!
// 6. Validate content based on file extension
instructions := instructions_response.result.trim_space()
// Step 2: Use morph model to apply instructions to original content
morph_prompt := generate_morph_prompt(original_content, instructions)
morph_response := ac.llms.llm_morph.chat_completion(
message: morph_prompt
temperature: args.temperature
max_completion_tokens: args.max_tokens
)!
new_content := morph_response.result.trim_space()
// Step 3: Validate content based on file extension
mut validation_error := ''
match mypath.ext()! {
// Create a temporary file for validation
file_ext := mypath.ext()!
mut temp_path := pathlib.get_file(
path: '${mypath.path}.validate_temp${file_ext}'
create: true
)!
temp_path.write(new_content)!
match file_ext {
'.v' {
validation_error = validate_vlang_content(mypath)!
validation_error = validate_vlang_content(temp_path)!
}
'.md' {
validation_error = validate_markdown_content(mypath)!
validation_error = validate_markdown_content(temp_path)!
}
'.yaml', '.yml' {
validation_error = validate_yaml_content(mypath)!
validation_error = validate_yaml_content(temp_path)!
}
'.json' {
validation_error = validate_json_content(mypath)!
validation_error = validate_json_content(temp_path)!
}
else {
// No specific validation for other file types
}
}
// Clean up temporary validation file
if temp_path.exists() {
temp_path.delete()!
}
if validation_error == '' {
// Validation passed - write new content
mypath.write(new_content)!
backup_path.delete()! // Remove backup on success
console.print_stdout('✓ Successfully modified ${mypath.str()} using model ${model_name}')
return
} else {
console.print_stderr('Validation failed for model ${model_name}. Error: ${validation_error}. Trying next model...')
console.print_stderr('✗ Validation failed for model ${model_name}. Error: ${validation_error}. Trying next model...')
}
}
// 8. If all fail, restore .backup and error
// Step 4: If all models fail, restore backup and error
original_backup := backup_path.read()!
mypath.write(original_backup)!
backup_path.delete()!
return error('All models failed to generate valid content. Original file restored.')
return error('All models failed to generate valid content. Original file restored from backup.')
}
// get_llm_client returns the appropriate LLM client for the given model enum
fn get_llm_client(mut ac AIClient, model LLMEnum) &openai.OpenAI {
return match model {
.maverick { ac.llms.llm_maverick }
.qwen { ac.llms.llm_qwen }
.embed { ac.llms.llm_embed }
.llm_120b { ac.llms.llm_120b }
.best { ac.llms.llm_best }
.flash { ac.llms.llm_flash }
.pro { ac.llms.llm_pro }
.morph { ac.llms.llm_morph }
.local { ac.llms.llm_local }
}
}
// generate_instruction_prompt creates the prompt for generating modification instructions
fn generate_instruction_prompt(content string, file_ext string, user_prompt string) string {
return 'You are a file modification assistant specializing in ${file_ext} files.
The user will provide a file and a modification request. Your task is to analyze the request and respond with ONLY clear, concise modification instructions.
Do NOT apply the modifications yourself. Just provide step-by-step instructions that could be applied to transform the file.
Original file content:
\`\`\`${file_ext}
${content}
\`\`\`
File type: ${file_ext}
User modification request:
${user_prompt}
Provide only the modification instructions. Be specific and clear. Format your response as a numbered list of changes to make.'
}
// generate_morph_prompt creates the prompt for the morph model to apply instructions
fn generate_morph_prompt(original_content string, instructions string) string {
return 'You are an expert code and file modifier. Your task is to apply modification instructions to existing file content.
Take the original file content and the modification instructions, then generate the modified version.
IMPORTANT: Return ONLY the modified file content. Do NOT include:
- Markdown formatting or code blocks
- Explanations or commentary
- "Here is the modified file:" prefixes
- Any text other than the actual modified content
Original file content:
\`\`\`
${original_content}
\`\`\`
Modification instructions to apply:
${instructions}
Return the complete modified file content:'
}

141
lib/ai/filemap/README.md Normal file
View File

@@ -0,0 +1,141 @@
# filemap Module
Parse directories or formatted strings into file maps with automatic ignore pattern support.
## Features
- 📂 Walk directories recursively and build file maps
- 🚫 Respect `.gitignore` and `.heroignore` ignore patterns with directory scoping
- 📝 Parse custom `===FILE:name===` format into file maps
- 📦 Export/write file maps to disk
- 🛡️ Robust, defensive parsing (handles spaces, variable `=` length, case-insensitive)
## Quick Start
### From Directory Path
```v
import incubaid.herolib.lib.ai.filemap
mut cw := filemap.new()
mut fm := cw.filemap_get(path: '/path/to/project')!
// Iterate files
for path, content in fm.content {
println('${path}: ${content.len} bytes')
}
```
### From Formatted String
```v
content_str := '
===FILE:main.v===
fn main() {
println("Hello!")
}
===FILE:utils/helper.v===
pub fn help() {}
===END===
'
mut cw := filemap.new()
mut fm := cw.parse(content_str)!
println(fm.get('main.v')!)
```
## FileMap Operations
```v
// Get file content
content := fm.get('path/to/file.txt')!
// Set/modify file
fm.set('new/file.txt', 'content here')
// Find files by prefix
files := fm.find('src/')
// Export to directory
fm.export('/output/dir')!
// Write updates to directory
fm.write('/project/dir')!
// Convert back to formatted string
text := fm.content()
```
## File Format
### Full Files
```
===FILE:path/to/file.txt===
File content here
Can span multiple lines
===END===
```
### Partial Content (for future morphing)
```
===FILECHANGE:src/models.v===
struct User {
id int
}
===END===
```
### Both Together
```
===FILE:main.v===
fn main() {}
===FILECHANGE:utils.v===
fn helper() {}
===END===
```
## Parsing Robustness
Parser handles variations:
```
===FILE:name.txt=== // Standard
== FILE : name.txt ==
===file:name.txt=== // Lowercase
==FILE:name.txt== // Different = count
```
## Error Handling
Errors are collected in `FileMap.errors`:
```v
mut fm := cw.filemap_get(content: str)!
if fm.errors.len > 0 {
for err in fm.errors {
println('Line ${err.linenr}: ${err.message}')
}
}
```
## Ignore Patterns
- Respects `.gitignore` and `.heroignore` in any parent directory
- Default patterns include `.git/`, `node_modules/`, `*.pyc`, etc.
- Use `/` suffix for directory patterns: `dist/`
- Use `*` for wildcards: `*.log`
- Lines starting with `#` are comments
Example `.heroignore`:
```
build/
*.tmp
.env
__pycache__/
```

24
lib/ai/filemap/factory.v Normal file
View File

@@ -0,0 +1,24 @@
module filemap
@[params]
pub struct FileMapArgs {
pub mut:
path string
content string
content_read bool = true // If false, file content not read from disk
// Include if matches any wildcard pattern (* = any sequence)
filter []string
// Exclude if matches any wildcard pattern
filter_ignore []string
}
// filemap_get creates FileMap from path or content string
pub fn filemap(args FileMapArgs) !FileMap {
if args.path != '' {
return filemap_get_from_path(args.path, args.content_read)!
} else if args.content != '' {
return filemap_get_from_content(args.content)!
} else {
return error('Either path or content must be provided')
}
}

24
lib/develop/codewalker/filemap.v → lib/ai/filemap/filemap.v
View File

@@ -1,15 +1,17 @@
module codewalker
module filemap
import incubaid.herolib.core.pathlib
// FileMap represents parsed file structure with content and changes
pub struct FileMap {
pub mut:
source string
content map[string]string
content_change map[string]string
errors []FMError
source string // Source path or origin
content map[string]string // Full file content by path
content_change map[string]string // Partial/change content by path
errors []FMError // Parse errors encountered
}
// content generates formatted string representation
pub fn (mut fm FileMap) content() string {
mut out := []string{}
for filepath, filecontent in fm.content {
@@ -24,7 +26,7 @@ pub fn (mut fm FileMap) content() string {
return out.join_lines()
}
// write in new location, all will be overwritten, will only work with full files, not changes
// export writes all FILE content to destination directory
pub fn (mut fm FileMap) export(path string) ! {
for filepath, filecontent in fm.content {
dest := '${path}/${filepath}'
@@ -33,7 +35,7 @@ pub fn (mut fm FileMap) export(path string) ! {
}
}
@[PARAMS]
@[params]
pub struct WriteParams {
path string
v_test bool = true
@@ -41,29 +43,31 @@ pub struct WriteParams {
python_test bool
}
// update the files as found in the folder and update them or create
// write updates files in destination directory (creates or overwrites)
pub fn (mut fm FileMap) write(path string) ! {
for filepath, filecontent in fm.content {
dest := '${path}/${filepath}'
// In future: validate language-specific formatting/tests before overwrite
mut filepathtowrite := pathlib.get_file(path: dest, create: true)!
filepathtowrite.write(filecontent)!
}
// TODO: phase 2, work with morphe to integrate change in the file
}
// get retrieves file content by path
pub fn (fm FileMap) get(relpath string) !string {
return fm.content[relpath] or { return error('File not found: ${relpath}') }
}
// set stores file content by path
pub fn (mut fm FileMap) set(relpath string, content string) {
fm.content[relpath] = content
}
// delete removes file from content map
pub fn (mut fm FileMap) delete(relpath string) {
fm.content.delete(relpath)
}
// find returns all paths matching prefix
pub fn (fm FileMap) find(path string) []string {
mut result := []string{}
for filepath, _ in fm.content {

363
lib/ai/filemap/filemap_test.v Normal file
View File

@@ -0,0 +1,363 @@
module filemap
import os
import incubaid.herolib.core.pathlib
fn test_parse_header_file() {
kind, name := parse_header('===FILE:main.v===')!
assert kind == BlockKind.file
assert name == 'main.v'
}
fn test_parse_header_file2() {
kind, name := parse_header('===FILE:main.v ===')!
assert kind == BlockKind.file
assert name == 'main.v'
}
fn test_parse_header_file3() {
kind, name := parse_header('=== FILE:main.v ===')!
assert kind == BlockKind.file
assert name == 'main.v'
}
fn test_parse_header_file4() {
kind, name := parse_header('== FILE: main.v =====')!
assert kind == BlockKind.file
assert name == 'main.v'
}
fn test_parse_header_filechange() {
kind, name := parse_header('===FILECHANGE:utils/helper.v===')!
assert kind == BlockKind.filechange
assert name == 'utils/helper.v'
}
fn test_parse_header_end() {
kind, _ := parse_header('===END===')!
assert kind == BlockKind.end
}
fn test_parse_header_with_spaces() {
kind, name := parse_header(' === FILE : config.yaml === ')!
assert kind == BlockKind.file
assert name == 'config.yaml'
}
fn test_parse_header_lowercase() {
kind, name := parse_header('===file:test.txt===')!
assert kind == BlockKind.file
assert name == 'test.txt'
}
fn test_parse_header_variable_equals() {
kind, name := parse_header('=FILE:path/file.v=')!
assert kind == BlockKind.file
assert name == 'path/file.v'
}
fn test_parse_header_end_lowercase() {
kind, _ := parse_header('===end===')!
assert kind == BlockKind.end
}
fn test_filemap_from_simple_content() {
content := '===FILE:main.v===
fn main() {
println("Hello, World!")
}
===END==='
fm := filemap_get_from_content(content)!
assert fm.content.len == 1
assert 'main.v' in fm.content
assert fm.content['main.v'].contains('println')
}
fn test_filemap_from_multiple_files() {
content := '===FILE:main.v===
fn main() {
println("Hello")
}
===FILE:utils/helper.v===
pub fn help() {
println("Helping")
}
===END==='
fm := filemap_get_from_content(content)!
assert fm.content.len == 2
assert 'main.v' in fm.content
assert 'utils/helper.v' in fm.content
}
fn test_filemap_with_filechange() {
content := '===FILE:config.v===
pub const version = "1.0"
===FILECHANGE:main.v===
fn main() {
println(version)
}
===END==='
fm := filemap_get_from_content(content)!
assert fm.content.len == 1
assert fm.content_change.len == 1
assert 'config.v' in fm.content
assert 'main.v' in fm.content_change
}
fn test_filemap_multiline_content() {
content := '===FILE:multiline.txt===
Line 1
Line 2
Line 3
===FILE:another.txt===
Another content
===END==='
fm := filemap_get_from_content(content)!
assert fm.content['multiline.txt'].contains('Line 1')
assert fm.content['multiline.txt'].contains('Line 2')
assert fm.content['multiline.txt'].contains('Line 3')
assert fm.content['another.txt'] == 'Another content'
}
fn test_filemap_get_method() {
content := '===FILE:test.v===
test content
===END==='
fm := filemap_get_from_content(content)!
result := fm.get('test.v')!
assert result == 'test content'
}
fn test_filemap_get_not_found() {
content := '===FILE:test.v===
content
===END==='
fm := filemap_get_from_content(content)!
result := fm.get('nonexistent.v') or {
assert err.msg().contains('File not found')
return
}
panic('Should have returned error')
}
fn test_filemap_set_method() {
mut fm := FileMap{}
fm.set('new/file.v', 'new content')
assert fm.content['new/file.v'] == 'new content'
}
fn test_filemap_delete_method() {
mut fm := FileMap{}
fm.set('file1.v', 'content1')
fm.set('file2.v', 'content2')
assert fm.content.len == 2
fm.delete('file1.v')
assert fm.content.len == 1
assert 'file2.v' in fm.content
assert 'file1.v' !in fm.content
}
fn test_filemap_find_method() {
mut fm := FileMap{}
fm.set('src/main.v', 'main')
fm.set('src/utils/helper.v', 'helper')
fm.set('test/test.v', 'test')
results := fm.find('src/')
assert results.len == 2
assert 'src/main.v' in results
assert 'src/utils/helper.v' in results
}
fn test_filemap_find_empty() {
mut fm := FileMap{}
fm.set('main.v', 'main')
results := fm.find('src/')
assert results.len == 0
}
fn test_filemap_from_path() {
// Create temporary test directory
tmpdir := os.temp_dir() + '/test_filemap_${os.getpid()}'
os.mkdir_all(tmpdir) or { panic(err) }
defer {
os.rmdir_all(tmpdir) or {}
}
// Create test files
os.mkdir_all('${tmpdir}/src') or { panic(err) }
os.mkdir_all('${tmpdir}/test') or { panic(err) }
os.write_file('${tmpdir}/main.v', 'fn main() {}')!
os.write_file('${tmpdir}/src/utils.v', 'pub fn help() {}')!
os.write_file('${tmpdir}/test/test.v', 'fn test() {}')!
fm := filemap_get_from_path(tmpdir, true)!
assert fm.content.len >= 3
assert 'main.v' in fm.content
assert fm.content['main.v'] == 'fn main() {}'
}
fn test_filemap_from_path_no_content() {
tmpdir := os.temp_dir() + '/test_filemap_nocontent_${os.getpid()}'
os.mkdir_all(tmpdir) or { panic(err) }
defer {
os.rmdir_all(tmpdir) or {}
}
os.mkdir_all('${tmpdir}/src') or { panic(err) }
os.write_file('${tmpdir}/main.v', 'fn main() {}')!
fm := filemap_get_from_path(tmpdir, false)!
assert fm.content.len >= 1
assert 'main.v' in fm.content
assert fm.content['main.v'] == ''
}
fn test_filemap_from_path_not_exists() {
result := filemap_get_from_path('/nonexistent/path/12345', true) or {
assert err.msg().contains('does not exist')
return
}
panic('Should have returned error for nonexistent path')
}
fn test_filemap_content_string() {
mut fm := FileMap{}
fm.set('file1.v', 'content1')
fm.set('file2.v', 'content2')
output := fm.content()
assert output.contains('===FILE:file1.v===')
assert output.contains('content1')
assert output.contains('===FILE:file2.v===')
assert output.contains('content2')
assert output.contains('===END===')
}
fn test_filemap_export() {
tmpdir := os.temp_dir() + '/test_filemap_export_${os.getpid()}'
os.mkdir_all(tmpdir) or { panic(err) }
defer {
os.rmdir_all(tmpdir) or {}
}
mut fm := FileMap{}
fm.set('main.v', 'fn main() {}')
fm.set('src/helper.v', 'pub fn help() {}')
fm.export(tmpdir)!
assert os.exists('${tmpdir}/main.v')
assert os.exists('${tmpdir}/src/helper.v')
assert os.read_file('${tmpdir}/main.v')! == 'fn main() {}'
}
fn test_filemap_write() {
tmpdir := os.temp_dir() + '/test_filemap_write_${os.getpid()}'
os.mkdir_all(tmpdir) or { panic(err) }
defer {
os.rmdir_all(tmpdir) or {}
}
mut fm := FileMap{}
fm.set('config.v', 'const version = "1.0"')
fm.set('models/user.v', 'struct User {}')
fm.write(tmpdir)!
assert os.exists('${tmpdir}/config.v')
assert os.exists('${tmpdir}/models/user.v')
}
fn test_filemap_factory_from_path() {
tmpdir := os.temp_dir() + '/test_factory_path_${os.getpid()}'
os.mkdir_all(tmpdir) or { panic(err) }
defer {
os.rmdir_all(tmpdir) or {}
}
os.write_file('${tmpdir}/test.v', 'fn test() {}')!
fm := filemap(path: tmpdir, content_read: true)!
assert 'test.v' in fm.content
}
fn test_filemap_factory_from_content() {
content := '===FILE:sample.v===
fn main() {}
===END==='
fm := filemap(content: content)!
assert 'sample.v' in fm.content
}
fn test_filemap_factory_requires_input() {
result := filemap(path: '', content: '') or {
assert err.msg().contains('Either path or content')
return
}
panic('Should have returned error')
}
fn test_filemap_parse_errors_content_before_file() {
content := 'Some text before file
===FILE:main.v===
content
===END==='
fm := filemap_get_from_content(content)!
assert fm.errors.len > 0
assert fm.errors[0].category == 'parse'
}
fn test_filemap_parse_errors_end_without_file() {
content := '===END==='
fm := filemap_get_from_content(content)!
assert fm.errors.len > 0
}
fn test_filemap_empty_content() {
content := ''
fm := filemap_get_from_content(content)!
assert fm.content.len == 0
}
fn test_filemap_complex_filenames() {
content := '===FILE:src/v_models/user_model.v===
pub struct User {}
===FILE:test/unit/user_test.v===
fn test_user() {}
===FILE:.config/settings.json===
{ "key": "value" }
===END==='
fm := filemap_get_from_content(content)!
assert 'src/v_models/user_model.v' in fm.content
assert 'test/unit/user_test.v' in fm.content
assert '.config/settings.json' in fm.content
}
fn test_filemap_whitespace_preservation() {
content := '===FILE:formatted.txt===
Line with spaces
Tab indented
Spaces indented
===END==='
fm := filemap_get_from_content(content)!
file_content := fm.content['formatted.txt']
assert file_content.contains(' spaces')
assert file_content.contains('\t')
}

114
lib/ai/filemap/ignore.v Normal file
View File

@@ -0,0 +1,114 @@
module filemap
import arrays
import os
import incubaid.herolib.core.pathlib
// Default ignore patterns based on .gitignore conventions
const default_gitignore = '
.git/
.svn/
.hg/
.bzr/
node_modules/
__pycache__/
*.py[cod]
*.so
.Python
build/
develop-eggs/
downloads/
eggs/
.eggs/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
.env
.venv
venv/
.tox/
.nox/
.coverage
.coveragerc
coverage.xml
*.cover
*.gem
*.pyc
.cache
.pytest_cache/
.mypy_cache/
.hypothesis/
.DS_Store
Thumbs.db
*.tmp
*.temp
*.log
'
// find_ignore_patterns collects all .gitignore patterns from current directory up to repository root
//
// Walks up the directory tree using parent_find_advanced to locate all .gitignore files,
// stopping when it encounters the .git directory (repository root).
// Patterns are collected from:
// 1. Default ignore patterns (built-in)
// 2. All .gitignore files found from current directory to repository root
// 3. Filter out comments (lines starting with '#') and empty lines
//
// Parameters:
// - start_path: Optional starting directory path (defaults to current working directory if empty)
//
// Returns:
// - Combined, sorted, unique ignore patterns from all sources
// - Error if path operations fail (file not found, permission denied, etc.)
//
// Examples:
// // Use current working directory
// patterns := find_ignore_patterns()!
//
// // Use specific project directory
// patterns := find_ignore_patterns('/home/user/myproject')!
pub fn find_ignore_patterns(start_path string) ![]string {
mut patterns := default_gitignore.split_into_lines()
// Use provided path or current working directory
mut search_from := start_path
if search_from == '' { // If an empty string was passed for start_path, use current working directory
search_from = os.getwd()
}
mut current_path := pathlib.get(search_from)
// Find all .gitignore files up the tree until we hit .git directory (repo root)
mut gitignore_paths := current_path.parent_find_advanced('.gitignore', '.git')!
// Read and collect patterns from all found .gitignore files
for mut gitignore_path in gitignore_paths {
if gitignore_path.is_file() {
content := gitignore_path.read() or {
// Skip files that can't be read (permission issues, etc.)
continue
}
gitignore_lines := content.split_into_lines()
for line in gitignore_lines {
trimmed := line.trim_space()
// Skip empty lines and comment lines
if trimmed != '' && !trimmed.starts_with('#') {
patterns << trimmed
}
}
}
}
// Sort and get unique patterns to remove duplicates
patterns.sort()
patterns = arrays.uniq(patterns)
return patterns
}

127
lib/ai/filemap/loaders.v Normal file
View File

@@ -0,0 +1,127 @@
module filemap
import incubaid.herolib.core.pathlib
// filemap_get_from_path reads directory and creates FileMap, respecting ignore patterns
fn filemap_get_from_path(path string, content_read bool) !FileMap {
mut dir := pathlib.get(path)
if !dir.exists() || !dir.is_dir() {
return error('Directory "${path}" does not exist')
}
mut fm := FileMap{
source: path
}
ignore_patterns := find_ignore_patterns(path)!
// List all files using pathlib with both default and custom ignore patterns
mut file_list := dir.list(
recursive: true
filter_ignore: ignore_patterns
)!
for mut file in file_list.paths {
if file.is_file() {
relpath := file.path_relative(path)!
if content_read {
content := file.read()!
fm.content[relpath] = content
} else {
fm.content[relpath] = ''
}
}
}
return fm
}
// filemap_get_from_content parses FileMap from string with ===FILE:name=== format
fn filemap_get_from_content(content string) !FileMap {
mut fm := FileMap{}
mut current_kind := BlockKind.end
mut filename := ''
mut block := []string{}
mut had_any_block := false
mut linenr := 0
for line in content.split_into_lines() {
linenr += 1
parsed_kind, parsed_name := parse_header(line)! // Call parse_header with the raw line
mut is_a_header_line := false
if parsed_kind == .file || parsed_kind == .filechange {
is_a_header_line = true
} else if parsed_kind == .end && line.trim_space().to_lower() == '===end===' {
// This is explicitly an END header
is_a_header_line = true
}
if is_a_header_line {
// Handle the header line (logic similar to current .file, .filechange, and .end blocks)
if parsed_kind == .end { // It's the explicit ===END===
if filename == '' {
if had_any_block {
fm.errors << FMError{
message: 'Unexpected END marker without active block'
linenr: linenr
category: 'parse'
}
} else {
fm.errors << FMError{
message: 'END found before any FILE block'
linenr: linenr
category: 'parse'
}
}
} else {
// Store current block
match current_kind {
.file { fm.content[filename] = block.join_lines() }
.filechange { fm.content_change[filename] = block.join_lines() }
else {}
}
filename = ''
block = []string{}
current_kind = .end
}
} else { // It's a FILE or FILECHANGE header
// Flush previous block if any
if filename != '' {
match current_kind {
.file { fm.content[filename] = block.join_lines() }
.filechange { fm.content_change[filename] = block.join_lines() }
else {}
}
}
filename = parsed_name
current_kind = parsed_kind
block = []string{}
had_any_block = true
}
} else {
// This is a content line (parse_header returned .end, but it wasn't '===END===')
if filename == '' && line.trim_space().len > 0 {
fm.errors << FMError{
message: "Content before first FILE block: '${line}'"
linenr: linenr
category: 'parse'
}
} else if filename != '' {
block << line
}
}
}
// Flush final block if any
if filename != '' {
match current_kind {
.file { fm.content[filename] = block.join_lines() }
.filechange { fm.content_change[filename] = block.join_lines() }
else {}
}
}
return fm
}

16
lib/ai/filemap/model.v Normal file
View File

@@ -0,0 +1,16 @@
module filemap
// BlockKind defines the type of block in parsed content
pub enum BlockKind {
file
filechange
end
}
pub struct FMError {
pub:
message string
linenr int
category string
filename string
}

44
lib/ai/filemap/parser.v Normal file
View File

@@ -0,0 +1,44 @@
module filemap
// parse_header robustly extracts block type and filename from header line
// Handles variable `=` count, spaces, and case-insensitivity
// Example: ` ===FILE: myfile.txt ===` → $(BlockKind.file, "myfile.txt")
fn parse_header(line string) !(BlockKind, string) {
cleaned := line.trim_space()
// Must have = and content
if !cleaned.contains('=') {
return BlockKind.end, ''
}
// Strip leading and trailing = (any count), preserving spaces between
mut content := cleaned.trim_left('=').trim_space()
content = content.trim_right('=').trim_space()
if content.len == 0 {
return BlockKind.end, ''
}
// Check for END marker
if content.to_lower() == 'end' {
return BlockKind.end, ''
}
// Parse FILE or FILECHANGE
if content.contains(':') {
kind_str := content.all_before(':').to_lower().trim_space()
filename := content.all_after(':').trim_space()
if filename.len < 1 {
return error('Invalid filename: empty after colon')
}
match kind_str {
'file' { return BlockKind.file, filename }
'filechange' { return BlockKind.filechange, filename }
else { return BlockKind.end, '' }
}
}
return BlockKind.end, ''
}

7
lib/ai/instructions/factory.v Normal file
View File

@@ -0,0 +1,7 @@
module instructions
import incubaid.herolib.core.texttools
__global (
instructions_cache map[string]string
)

39
lib/ai/instructions/hero.v Normal file
View File

@@ -0,0 +1,39 @@
module heromodels
import incubaid.herolib.develop.gittools
import incubaid.herolib.core.pathlib
import incubaid.herolib.lib.develop.codewalker
pub fn aiprompts_path() !string {
return instructions_cache['aiprompts_path'] or {
mypath := gittools.path(
git_url: 'https://github.com/Incubaid/herolib/tree/development/aiprompts'
)!.path
instructions_cache['aiprompts_path'] = mypath
mypath
}
}
pub fn ai_instructions_hero_models() !string {
path := '${aiprompts_path()!}/ai_instructions_hero_models.md'
mut ppath := pathlib.get_file(path: path, create: false)!
return ppath.read()!
}
pub fn ai_instructions_vlang_herolib_core() !string {
path := '${aiprompts_path()!}/vlang_herolib_core.md'
mut ppath := pathlib.get_file(path: path, create: false)!
return ppath.read()!
}
pub fn ai_instructions_herolib_core_all() !string {
path := '${aiprompts_path()!}/herolib_core'
mut cw := codewalker.new()!
mut filemap := cw.filemap_get(
path: path
)!
println(false)
$dbg;
return filemap.content()
}

9
lib/aiflows/calendar/actions.v Normal file
View File

@@ -0,0 +1,9 @@
module flow_calendar
import incubaid.herolib.hero.heromodels
import incubaid.herolib.core.flows
pub fn calendar_delete(mut s flows.Step) ! {
// get heromodels
mut m := heromodels.get('coordinator_${s.coordinator.name}')!
}

20
lib/aiflows/calendar/start.v Normal file
View File

@@ -0,0 +1,20 @@
module flow_calendar
import incubaid.herolib.hero.heromodels
import incubaid.herolib.core.flows
type CoordinatorProxy = flows.Coordinator
pub fn start(mut c flows.Coordinator, prompt string) ! {
// init the heromodels, define well chosen name, needed to call later
mut m := heromodels.new(redis: c.redis, name: 'coordinator_${c.name}')!
mut step_triage := c.step_new(
context: {
'prompt': prompt
}
f: triage
)!
c.run()!
}

13
lib/aiflows/calendar/triage.v Normal file
View File

@@ -0,0 +1,13 @@
module flow_calendar
import incubaid.herolib.hero.heromodels
import incubaid.herolib.core.flows
pub fn triage(mut s flows.Step) ! {
prompt := s.context['prompt'] or { panic("can't find prompt context in step:\n${s}") }
response := s.coordinator.ai.llms.llm_maverick.chat_completion(
message: prompt
temperature: 0.5
max_completion_tokens: 5000
)!
}

4
lib/clients/openai/completions.v
View File

@@ -84,8 +84,10 @@ pub fn (mut f OpenAI) chat_completion(args_ CompletionArgs) !ChatCompletion {
m.messages << mr
}
data := json.encode(m)
// println('data: ${data}')
println('data: ${data}')
mut conn := f.connection()!
println(conn)
r := conn.post_json_str(prefix: 'chat/completions', data: data)!
res := json.decode(ChatCompletionRaw, r)!

49
lib/clients/openai/embeddings.v Normal file
View File

@@ -0,0 +1,49 @@
module openai
import json
// pub enum EmbeddingModel {
// text_embedding_ada
// }
// fn embedding_model_str(e EmbeddingModel) string {
// return match e {
// .text_embedding_ada {
// 'text-embedding-ada-002'
// }
// }
// }
@[params]
pub struct EmbeddingCreateRequest {
pub mut:
input []string @[required]
model string
user string
}
pub struct Embedding {
pub mut:
object string
embedding []f32
index int
}
pub struct EmbeddingResponse {
pub mut:
object string
data []Embedding
model string
usage Usage
}
pub fn (mut f OpenAI) embed(args_ EmbeddingCreateRequest) !EmbeddingResponse {
mut args := args_
if args.model == '' {
args.model = f.model_default
}
data := json.encode(args)
mut conn := f.connection()!
r := conn.post_json_str(prefix: 'embeddings', data: data)!
return json.decode(EmbeddingResponse, r)!
}

17
lib/clients/openai/embeddings/README.md
View File

@@ -1,17 +0,0 @@
# Quick Example: Creating Embeddings
```v
import incubaid.herolib.clients.openai
mut client:= openai.get()! //will be the default client, key is in `AIKEY` on environment variable or `OPENROUTER_API_KEY`
text_to_embed := 'The quick brown fox jumps over the lazy dog.'
resp := client.embeddings.create_embedding(
input: text_to_embed,
model: 'text-embedding-ada-002'
)!
```

59
lib/clients/openai/embeddings/embeddings.v
View File

@@ -1,59 +0,0 @@
module embeddings
import json
import incubaid.herolib.clients.openai { OpenAI, Usage }
type OpenAIAlias = OpenAI
pub enum EmbeddingModel {
text_embedding_ada
}
fn embedding_model_str(e EmbeddingModel) string {
return match e {
.text_embedding_ada {
'text-embedding-ada-002'
}
}
}
@[params]
pub struct EmbeddingCreateArgs {
input []string @[required]
model EmbeddingModel @[required]
user string
}
pub struct EmbeddingCreateRequest {
input []string
model string
user string
}
pub struct Embedding {
pub mut:
object string
embedding []f32
index int
}
pub struct EmbeddingResponse {
pub mut:
object string
data []Embedding
model string
usage Usage
}
pub fn (mut f OpenAIAlias) create_embeddings(args EmbeddingCreateArgs) !EmbeddingResponse {
req := EmbeddingCreateRequest{
input: args.input
model: embedding_model_str(args.model)
user: args.user
}
data := json.encode(req)
mut conn := f.connection()!
r := conn.post_json_str(prefix: 'embeddings', data: data)!
return json.decode(EmbeddingResponse, r)!
}

6
lib/clients/openai/openai_model.v
View File

@@ -52,9 +52,9 @@ fn obj_init(mycfg_ OpenAI) !OpenAI {
}
}
}
if mycfg.api_key == '' {
return error('OpenAI client "${mycfg.name}" missing api_key')
}
// if mycfg.api_key == '' {
// return error('OpenAI client "${mycfg.name}" missing api_key')
// }
return mycfg
}

299
lib/core/code/README.md
View File

@@ -1,73 +1,270 @@
# Code Model
A set of models that represent code, such as structs and functions. The motivation behind this module is to provide a more generic, and lighter alternative to v.ast code models, that can be used for code parsing and code generation across multiple languages.
A comprehensive module for parsing, analyzing, and generating V code. The Code Model provides lightweight, language-agnostic structures to represent code elements like structs, functions, imports, and types.
## Using Codemodel
## Overview
While the models in this module can be used in any domain, the models here are used extensively in the modules [codeparser](../codeparser/) and codegen (under development). Below are examples on how codemodel can be used for parsing and generating code.
## Code parsing with codemodel
The `code` module is useful for:
As shown in the example below, the codemodels returned by the parser can be used to infer information about the code written
- **Code Parsing**: Parse V files into structured models
- **Code Analysis**: Extract information about functions, structs, and types
- **Code Generation**: Generate V code from models using `vgen()`
- **Static Analysis**: Inspect and traverse code using language utilities
- **Documentation Generation**: Serialize code into other formats (JSON, Markdown, etc.)
```js
code := codeparser.parse("somedir") // code is a list of code models
## Core Components
num_functions := code.filter(it is Function).len
structs := code.filter(it is Struct)
println("This directory has ${num_functions} functions")
println('The directory has the structs: ${structs.map(it.name)}')
### Code Structures (Models)
- **`Struct`**: Represents V struct definitions with fields, visibility, and generics
- **`Function`**: Represents functions/methods with parameters, return types, and bodies
- **`Interface`**: Represents V interface definitions
- **`VFile`**: Represents a complete V file with module, imports, constants, and items
- **`Module`**: Represents a V module with nested files and folders
- **`Import`**: Represents import statements
- **`Param`**: Represents function parameters with types and modifiers
- **`Type`**: Union type supporting arrays, maps, results, objects, and basic types
- **`Const`**: Represents constant definitions
### Type System
The `Type` union supports:
- Basic types: `String`, `Boolean`, `Integer` (signed/unsigned, 8/16/32/64-bit)
- Composite types: `Array`, `Map`, `Object`
- Function types: `Function`
- Result types: `Result` (for error handling with `!`)
- Aliases: `Alias`
## Usage Examples
### Parsing a V File
```v
import incubaid.herolib.core.code
import os
// Read and parse a V file
content := os.read_file('path/to/file.v')!
vfile := code.parse_vfile(content)!
// Access parsed elements
println('Module: ${vfile.mod}')
println('Imports: ${vfile.imports.len}')
println('Structs: ${vfile.structs().len}')
println('Functions: ${vfile.functions().len}')
```
or can be used as intermediate structures to serialize code into some other format:
### Analyzing Structs
```js
code_md := ''
```v
import incubaid.herolib.core.code
// describes the struct in markdown format
for struct in structs {
code_md += '# ${struct.name}'
code_md += 'Type: ${struct.typ.symbol()}'
code_md += '## Fields:'
for field in struct.fields {
code_md += '- ${field.name}'
// Parse a struct definition
struct_code := 'pub struct User {
pub:
name string
age int
}'
vfile := code.parse_vfile(struct_code)!
structs := vfile.structs()
for struct_ in structs {
println('Struct: ${struct_.name}')
println(' Is public: ${struct_.is_pub}')
for field in struct_.fields {
println(' Field: ${field.name} (${field.typ.symbol()})')
}
}
```
The [openrpc/docgen](../openrpc/docgen/) module demonstrates a good use case, where codemodels are serialized into JSON schema's, to generate an OpenRPC description document from a client in v.## V Language Utilities
The `vlang_utils.v` file provides a set of utility functions for working with V language files and code. These utilities are useful for:
1. **File Operations**
- `list_v_files(dir string) ![]string` - Lists all V files in a directory, excluding generated files
- `get_module_dir(mod string) string` - Converts a V module path to a directory path
2. **Code Inspection and Analysis**
- `get_function_from_file(file_path string, function_name string) !string` - Extracts a function definition from a file
- `get_function_from_module(module_path string, function_name string) !string` - Searches for a function across all files in a module
- `get_type_from_module(module_path string, type_name string) !string` - Searches for a type definition across all files in a module
3. **V Language Tools**
- `vtest(fullpath string) !string` - Runs V tests on files or directories
- `vvet(fullpath string) !string` - Runs V vet on files or directories
### Example Usage
### Analyzing Functions
```v
// Find and extract a function definition
function_def := code.get_function_from_module('/path/to/module', 'my_function') or {
eprintln('Could not find function: ${err}')
return
}
println(function_def)
import incubaid.herolib.core.code
// Run tests on a directory
test_results := code.vtest('/path/to/module') or {
eprintln('Tests failed: ${err}')
return
fn_code := 'pub fn greet(name string) string {
return "Hello, \${name}!"
}'
vfile := code.parse_vfile(fn_code)!
functions := vfile.functions()
for func in functions {
println('Function: ${func.name}')
println(' Public: ${func.is_pub}')
println(' Parameters: ${func.params.len}')
println(' Returns: ${func.result.typ.symbol()}')
}
println(test_results)
```
These utilities are particularly useful when working with code generation, static analysis, or when building developer tools that need to inspect V code.
### Code Generation
```v
import incubaid.herolib.core.code
// Create a struct model
my_struct := code.Struct{
name: 'Person'
is_pub: true
fields: [
code.StructField{
name: 'name'
typ: code.type_from_symbol('string')
is_pub: true
},
code.StructField{
name: 'age'
typ: code.type_from_symbol('int')
is_pub: true
}
]
}
// Generate V code from the model
generated_code := my_struct.vgen()
println(generated_code)
// Output: pub struct Person { ... }
```
### V Language Utilities
```v
import incubaid.herolib.core.code
// List all V files in a directory (excludes generated files ending with _.v)
v_files := code.list_v_files('/path/to/module')!
// Get a specific function from a module
func := code.get_function_from_module('/path/to/module', 'my_function')!
println('Found function: ${func.name}')
// Get a type definition from a module
type_def := code.get_type_from_module('/path/to/module', 'MyStruct')!
println(type_def)
// Run V tests
test_results := code.vtest('/path/to/module')!
```
### Working With Modules and Files
```v
import incubaid.herolib.core.code
// Create a module structure
my_module := code.Module{
name: 'mymodule'
description: 'My awesome module'
version: '1.0.0'
license: 'apache2'
files: [
code.VFile{
name: 'structs'
mod: 'mymodule'
// ... add items
}
]
}
// Write module to disk
write_opts := code.WriteOptions{
overwrite: false
format: true
compile: false
}
my_module.write('/output/path', write_opts)!
```
### Advanced Features
### Custom Code Generation
```v
import incubaid.herolib.core.code
// Generate a function call from a Function model
func := code.Function{
name: 'calculate'
params: [
code.Param{ name: 'x', typ: code.type_from_symbol('int') },
code.Param{ name: 'y', typ: code.type_from_symbol('int') }
]
result: code.Param{ typ: code.type_from_symbol('int') }
}
call := func.generate_call(receiver: 'calculator')!
// Output: result := calculator.calculate(...)
```
### Type Conversion
```v
import incubaid.herolib.core.code
// Convert from type symbol to Type model
t := code.type_from_symbol('[]string')
// Get the V representation
v_code := t.vgen() // Output: "[]string"
// Get the TypeScript representation
ts_code := t.typescript() // Output: "string[]"
// Get the symbol representation
symbol := t.symbol() // Output: "[]string"
```
## Complete Example
See the working example at **`examples/core/code/code_parser.vsh`** for a complete demonstration of:
- Listing V files in a directory
- Parsing multiple V files
- Extracting and analyzing structs and functions
- Summarizing module contents
Run it with:
```bash
vrun ~/code/github/incubaid/herolib/examples/core/code/code_parser.vsh
```
## Coding Instructions
When using the Code module:
1. **Always parse before analyzing**: Use `parse_vfile()`, `parse_struct()`, or `parse_function()` to create models from code strings
2. **Use type filters**: Filter code items by type using `.filter(it is StructType)` pattern
3. **Check visibility**: Always verify `is_pub` flag when examining public API
4. **Handle errors**: Code parsing can fail; always use `!` or `or` blocks
5. **Generate code carefully**: Use `WriteOptions` to control formatting, compilation, and testing
6. **Use language utilities**: Prefer `get_function_from_module()` over manual file searching
7. **Cache parsed results**: Store `VFile` objects if you need to access them multiple times
8. **Document generated code**: Add descriptions to generated structs and functions
## API Reference
### Parsing Functions
- `parse_vfile(code string) !VFile` - Parse an entire V file
- `parse_struct(code string) !Struct` - Parse a struct definition
- `parse_function(code string) !Function` - Parse a function definition
- `parse_param(code string) !Param` - Parse a parameter
- `parse_type(type_str string) Type` - Parse a type string
- `parse_const(code string) !Const` - Parse a constant
- `parse_import(code string) Import` - Parse an import statement
### Code Generation
- `vgen(code []CodeItem) string` - Generate V code from code items
- `Struct.vgen() string` - Generate struct V code
- `Function.vgen() string` - Generate function V code
- `Interface.vgen() string` - Generate interface V code
- `Import.vgen() string` - Generate import statement
### Language Utilities
- `list_v_files(dir string) ![]string` - List V files in directory
- `get_function_from_module(module_path string, name string) !Function` - Find function
- `get_type_from_module(module_path string, name string) !string` - Find type definition
- `get_module_dir(mod string) string` - Convert module name to directory path

3
lib/core/code/example.v
View File

@@ -1,3 +0,0 @@
module code
pub type Value = string

247
lib/core/code/improvements.md
View File

@@ -1,247 +0,0 @@
# Code Review and Improvement Plan for HeroLib Code Module
## Overview
The HeroLib `code` module provides utilities for parsing and generating V language code. It's designed to be a lightweight alternative to `v.ast` for code analysis and generation across multiple languages. While the module has good foundational structure, there are several areas that need improvement.
## Issues Identified
### 1. Incomplete TypeScript Generation Support
- The `typescript()` method exists in some models but lacks comprehensive implementation
- Missing TypeScript generation for complex types (arrays, maps, results)
- No TypeScript interface generation for structs
### 2. Template System Issues
- Some templates are empty (e.g., `templates/function/method.py`, `templates/comment/comment.py`)
- Template usage is inconsistent across the codebase
- No clear separation between V and other language templates
### 3. Missing Parser Documentation Examples
- README.md mentions codeparser but doesn't show how to use the parser from this module
- No clear examples of parsing V files or modules
### 4. Incomplete Type Handling
- The `parse_type` function doesn't handle all V language types comprehensively
- Missing support for function types, sum types, and complex generics
- No handling of optional types (`?Type`)
### 5. Code Structure and Consistency
- Some functions lack proper error handling
- Inconsistent naming conventions in test files
- Missing documentation for several key functions
## Improvement Plan
### 1. Complete TypeScript Generation Implementation
**What needs to be done:**
- Implement comprehensive TypeScript generation in `model_types.v`
- Add TypeScript generation for all type variants
- Create proper TypeScript interface generation in `model_struct.v`
**Specific fixes:**
```v
// In model_types.v, improve the typescript() method:
pub fn (t Type) typescript() string {
return match t {
Map { 'Record<string, ${t.typ.typescript()}>' }
Array { '${t.typ.typescript()}[]' }
Object { t.name }
Result { 'Promise<${t.typ.typescript()}>' } // Better representation for async operations
Boolean { 'boolean' }
Integer { 'number' }
Alias { t.name }
String { 'string' }
Function { '(...args: any[]) => any' } // More appropriate for function types
Void { 'void' }
}
}
// In model_struct.v, improve the typescript() method:
pub fn (s Struct) typescript() string {
name := texttools.pascal_case(s.name)
fields := s.fields.map(it.typescript()).join('\n ')
return 'export interface ${name} {\n ${fields}\n}'
}
```
### 2. Fix Template System
**What needs to be done:**
- Remove empty Python template files
- Ensure all templates are properly implemented
- Add template support for other languages
**Specific fixes:**
- Delete `templates/function/method.py` and `templates/comment/comment.py` if they're not needed
- Add proper TypeScript templates for struct and interface generation
- Create consistent template naming conventions
### 3. Improve Parser Documentation
**What needs to be done:**
- Add clear examples in README.md showing how to use the parser
- Document the parsing functions with practical examples
**Specific fixes:**
Add to README.md:
```markdown
## Parsing V Code
The code module provides utilities to parse V code into structured models:
```v
import incubaid.herolib.core.code
// Parse a V file
content := os.read_file('example.v') or { panic(err) }
vfile := code.parse_vfile(content) or { panic(err) }
// Access parsed information
println('Module: ${vfile.mod}')
println('Number of functions: ${vfile.functions().len}')
println('Number of structs: ${vfile.structs().len}')
// Parse individual components
function := code.parse_function(fn_code_string) or { panic(err) }
struct_ := code.parse_struct(struct_code_string) or { panic(err) }
```
### 4. Complete Type Handling
**What needs to be done:**
- Extend `parse_type` to handle more complex V types
- Add support for optional types (`?Type`)
- Improve generic type parsing
**Specific fixes:**
```v
// In model_types.v, enhance parse_type function:
pub fn parse_type(type_str string) Type {
mut type_str_trimmed := type_str.trim_space()
// Handle optional types
if type_str_trimmed.starts_with('?') {
return Optional{parse_type(type_str_trimmed.all_after('?'))}
}
// Handle function types
if type_str_trimmed.starts_with('fn ') {
// Parse function signature
return Function{}
}
// Handle sum types
if type_str_trimmed.contains('|') {
types := type_str_trimmed.split('|').map(parse_type(it.trim_space()))
return Sum{types}
}
// Existing parsing logic...
}
```
### 5. Code Structure Improvements
**What needs to be done:**
- Add proper error handling to all parsing functions
- Standardize naming conventions
- Improve documentation consistency
**Specific fixes:**
- Add error checking in `parse_function`, `parse_struct`, and other parsing functions
- Ensure all public functions have clear documentation comments
- Standardize test function names
## Module Generation to Other Languages
### Current Implementation
The current code shows basic TypeScript generation support, but it's incomplete. The generation should:
1. **Support multiple languages**: The code structure allows for multi-language generation, but only TypeScript has partial implementation
2. **Use templates consistently**: All language generation should use the template system
3. **Separate language-specific code**: Each language should have its own generation module
### What Needs to Move to Other Modules
**TypeScript Generation Module:**
- Move all TypeScript-specific generation code to a new `typescript` module
- Create TypeScript templates for structs, interfaces, and functions
- Add proper TypeScript formatting support
**Example Structure:**
```
lib/core/code/
├── model_types.v # Core type models (language agnostic)
├── model_struct.v # Core struct/function models (language agnostic)
└── typescript/ # TypeScript-specific generation
├── generator.v # TypeScript generation logic
└── templates/ # TypeScript templates
```
### Parser Usage Examples (to add to README.md)
```v
// Parse a V file into a structured representation
content := os.read_file('mymodule/example.v') or { panic(err) }
vfile := code.parse_vfile(content)!
// Extract all functions
functions := vfile.functions()
println('Found ${functions.len} functions')
// Extract all structs
structs := vfile.structs()
for s in structs {
println('Struct: ${s.name}')
for field in s.fields {
println(' Field: ${field.name} (${field.typ.symbol()})')
}
}
// Find a specific function
if greet_fn := vfile.get_function('greet') {
println('Found function: ${greet_fn.name}')
println('Parameters: ${greet_fn.params.map(it.name)}')
println('Returns: ${greet_fn.result.typ.symbol()}')
}
// Parse a function from string
fn_code := '
pub fn add(a int, b int) int {
return a + b
}
'
function := code.parse_function(fn_code)!
println('Parsed function: ${function.name}')
```
## Summary of Required Actions
1. **Implement complete TypeScript generation** across all model types
2. **Remove empty template files** and organize templates properly
3. **Enhance type parsing** to handle optional types, function types, and sum types
4. **Add comprehensive parser documentation** with practical examples to README.md
5. **Create language-specific generation modules** to separate concerns
6. **Improve error handling** in all parsing functions
7. **Standardize documentation and naming** conventions across the module
These improvements will make the code module more robust, easier to use, and better prepared for multi-language code generation.

16
lib/core/code/model.v
View File

@@ -11,6 +11,7 @@ pub type CodeItem = Alias
| Struct
| Sumtype
| Interface
| Enum
// item for adding custom code in
pub struct CustomCode {
@@ -31,6 +32,21 @@ pub:
types []Type
}
pub struct Enum {
pub mut:
name string
description string
is_pub bool
values []EnumValue
}
pub struct EnumValue {
pub:
name string
value string
description string
}
pub struct Attribute {
pub:
name string // [name]

1
lib/core/code/model_const.v
View File

@@ -1,6 +1,7 @@
module code
pub struct Const {
pub mut:
name string
value string
}

96
lib/core/code/model_enum.v Normal file
View File

@@ -0,0 +1,96 @@
module code
pub fn parse_enum(code_ string) !Enum {
mut lines := code_.split_into_lines()
mut comment_lines := []string{}
mut enum_lines := []string{}
mut in_enum := false
mut enum_name := ''
mut is_pub := false
for line in lines {
trimmed := line.trim_space()
if !in_enum && trimmed.starts_with('//') {
comment_lines << trimmed.trim_string_left('//').trim_space()
} else if !in_enum && (trimmed.starts_with('enum ') || trimmed.starts_with('pub enum ')) {
in_enum = true
enum_lines << line
// Extract enum name
is_pub = trimmed.starts_with('pub ')
mut name_part := if is_pub {
trimmed.trim_string_left('pub enum ').trim_space()
} else {
trimmed.trim_string_left('enum ').trim_space()
}
if name_part.contains('{') {
enum_name = name_part.all_before('{').trim_space()
} else {
enum_name = name_part
}
} else if in_enum {
enum_lines << line
if trimmed.starts_with('}') {
break
}
}
}
if enum_name == '' {
return error('Invalid enum format: could not extract enum name')
}
// Process enum values
mut values := []EnumValue{}
for i := 1; i < enum_lines.len - 1; i++ {
line := enum_lines[i].trim_space()
// Skip empty lines and comments
if line == '' || line.starts_with('//') {
continue
}
// Parse enum value
parts := line.split('=').map(it.trim_space())
value_name := parts[0]
value_content := if parts.len > 1 { parts[1] } else { '' }
values << EnumValue{
name: value_name
value: value_content
}
}
// Process comments into description
description := comment_lines.join('\n')
return Enum{
name: enum_name
description: description
is_pub: is_pub
values: values
}
}
pub fn (e Enum) vgen() string {
prefix := if e.is_pub { 'pub ' } else { '' }
comments := if e.description.trim_space() != '' {
'// ${e.description.trim_space()}\n'
} else {
''
}
mut values_str := ''
for value in e.values {
if value.value != '' {
values_str += '\n\t${value.name} = ${value.value}'
} else {
values_str += '\n\t${value.name}'
}
}
return '${comments}${prefix}enum ${e.name} {${values_str}\n}'
}

2
lib/core/code/model_example.v
View File

@@ -6,4 +6,4 @@ pub struct Example {
result Value
}
// pub type Value = string
pub type Value = string

33
lib/core/code/model_file.v
View File

@@ -165,8 +165,16 @@ pub fn (file VFile) structs() []Struct {
return file.items.filter(it is Struct).map(it as Struct)
}
pub fn (file VFile) enums() []Enum {
return file.items.filter(it is Enum).map(it as Enum)
}
pub fn (file VFile) interfaces() []Interface {
return file.items.filter(it is Interface).map(it as Interface)
}
// parse_vfile parses V code into a VFile struct
// It extracts the module name, imports, constants, structs, and functions
// It extracts the module name, imports, constants, structs, functions, enums and interfaces
pub fn parse_vfile(code string) !VFile {
mut vfile := VFile{
content: code
@@ -195,7 +203,7 @@ pub fn parse_vfile(code string) !VFile {
// Extract constants
vfile.consts = parse_consts(code) or { []Const{} }
// Split code into chunks for parsing structs and functions
// Split code into chunks for parsing structs, functions, enums, and interfaces
mut chunks := []string{}
mut current_chunk := ''
mut brace_count := 0
@@ -211,9 +219,12 @@ pub fn parse_vfile(code string) !VFile {
continue
}
// Check for struct or function start
// Check for struct, enum, interface or function start
if (trimmed.starts_with('struct ') || trimmed.starts_with('pub struct ')
|| trimmed.starts_with('fn ') || trimmed.starts_with('pub fn ')) && !in_struct_or_fn {
|| trimmed.starts_with('enum ') || trimmed.starts_with('pub enum ')
|| trimmed.starts_with('interface ')
|| trimmed.starts_with('pub interface ') || trimmed.starts_with('fn ')
|| trimmed.starts_with('pub fn ')) && !in_struct_or_fn {
in_struct_or_fn = true
current_chunk = comment_block.join('\n')
if current_chunk != '' {
@@ -238,7 +249,7 @@ pub fn parse_vfile(code string) !VFile {
continue
}
// Add line to current chunk if we're inside a struct or function
// Add line to current chunk if we're inside a struct, enum, interface or function
if in_struct_or_fn {
current_chunk += '\n' + line
@@ -249,7 +260,7 @@ pub fn parse_vfile(code string) !VFile {
brace_count -= line.count('}')
}
// Check if we've reached the end of the struct or function
// Check if we've reached the end
if brace_count == 0 {
chunks << current_chunk
current_chunk = ''
@@ -269,6 +280,16 @@ pub fn parse_vfile(code string) !VFile {
continue
}
vfile.items << struct_obj
} else if trimmed.contains('enum ') || trimmed.contains('pub enum ') {
// Parse enum
enum_obj := parse_enum(chunk) or {
// Skip invalid enums
continue
}
vfile.items << enum_obj
} else if trimmed.contains('interface ') || trimmed.contains('pub interface ') {
// Parse interface - TODO: implement when needed
continue
} else if trimmed.contains('fn ') || trimmed.contains('pub fn ') {
// Parse function
fn_obj := parse_function(chunk) or {

55
lib/core/code/model_types.v
View File

@@ -237,12 +237,21 @@ pub fn (t Type) empty_value() string {
// parse_type parses a type string into a Type struct
pub fn parse_type(type_str string) Type {
println('Parsing type string: "${type_str}"')
mut type_str_trimmed := type_str.trim_space()
mut type_str_cleaned := type_str.trim_space()
// Remove inline comments
if type_str_cleaned.contains('//') {
type_str_cleaned = type_str_cleaned.all_before('//').trim_space()
}
// Remove default values
if type_str_cleaned.contains('=') {
type_str_cleaned = type_str_cleaned.all_before('=').trim_space()
}
// Handle struct definitions by extracting just the struct name
if type_str_trimmed.contains('struct ') {
lines := type_str_trimmed.split_into_lines()
if type_str_cleaned.contains('struct ') {
lines := type_str_cleaned.split_into_lines()
for line in lines {
if line.contains('struct ') {
mut struct_name := ''
@@ -252,76 +261,74 @@ pub fn parse_type(type_str string) Type {
struct_name = line.all_after('struct ').all_before('{')
}
struct_name = struct_name.trim_space()
println('Extracted struct name: "${struct_name}"')
return Object{struct_name}
}
}
}
// Check for simple types first
if type_str_trimmed == 'string' {
if type_str_cleaned == 'string' {
return String{}
} else if type_str_trimmed == 'bool' || type_str_trimmed == 'boolean' {
} else if type_str_cleaned == 'bool' || type_str_cleaned == 'boolean' {
return Boolean{}
} else if type_str_trimmed == 'int' {
} else if type_str_cleaned == 'int' {
return Integer{}
} else if type_str_trimmed == 'u8' {
} else if type_str_cleaned == 'u8' {
return Integer{
bytes: 8
signed: false
}
} else if type_str_trimmed == 'u16' {
} else if type_str_cleaned == 'u16' {
return Integer{
bytes: 16
signed: false
}
} else if type_str_trimmed == 'u32' {
} else if type_str_cleaned == 'u32' {
return Integer{
bytes: 32
signed: false
}
} else if type_str_trimmed == 'u64' {
} else if type_str_cleaned == 'u64' {
return Integer{
bytes: 64
signed: false
}
} else if type_str_trimmed == 'i8' {
} else if type_str_cleaned == 'i8' {
return Integer{
bytes: 8
}
} else if type_str_trimmed == 'i16' {
} else if type_str_cleaned == 'i16' {
return Integer{
bytes: 16
}
} else if type_str_trimmed == 'i32' {
} else if type_str_cleaned == 'i32' {
return Integer{
bytes: 32
}
} else if type_str_trimmed == 'i64' {
} else if type_str_cleaned == 'i64' {
return Integer{
bytes: 64
}
}
// Check for array types
if type_str_trimmed.starts_with('[]') {
elem_type := type_str_trimmed.all_after('[]')
if type_str_cleaned.starts_with('[]') {
elem_type := type_str_cleaned.all_after('[]')
return Array{parse_type(elem_type)}
}
// Check for map types
if type_str_trimmed.starts_with('map[') && type_str_trimmed.contains(']') {
value_type := type_str_trimmed.all_after(']')
if type_str_cleaned.starts_with('map[') && type_str_cleaned.contains(']') {
value_type := type_str_cleaned.all_after(']')
return Map{parse_type(value_type)}
}
// Check for result types
if type_str_trimmed.starts_with('!') {
result_type := type_str_trimmed.all_after('!')
if type_str_cleaned.starts_with('!') {
result_type := type_str_cleaned.all_after('!')
return Result{parse_type(result_type)}
}
// If no other type matches, treat as an object/struct type
println('Treating as object type: "${type_str_trimmed}"')
return Object{type_str_trimmed}
return Object{type_str_cleaned}
}

280
lib/core/codegenerator/codegenerator.v Normal file
View File

@@ -0,0 +1,280 @@
module codegenerator
import incubaid.herolib.core.codeparser
import incubaid.herolib.core.pathlib
import incubaid.herolib.core.code
import incubaid.herolib.core.texttools
import os
pub struct CodeGenerator {
pub mut:
parser codeparser.CodeParser
output_dir string
format bool
}
// generate_all generates markdown docs for all modules
pub fn (mut gen CodeGenerator) generate_all() ! {
modules := gen.parser.list_modules()
for module_name in modules {
gen.generate_module(module_name)!
}
}
// generate_module generates markdown for a single module
pub fn (mut gen CodeGenerator) generate_module(module_name string) ! {
md := gen.module_to_markdown(module_name)!
// Convert module name to filename: incubaid.herolib.core.code -> code___core___code.md
filename := gen.module_to_filename(module_name)
filepath := os.join_path(gen.output_dir, filename)
mut file := pathlib.get_file(path: filepath, create: true)!
file.write(md)!
}
// module_to_markdown generates complete markdown for a module
pub fn (gen CodeGenerator) module_to_markdown(module_name string) !string {
module_obj := gen.parser.find_module(module_name)!
mut md := ''
// Use template for module header
md += $tmpl('templates/module.md.template')
// Imports section
imports := gen.parser.list_imports(module_name)
if imports.len > 0 {
md += gen.imports_section(imports)
}
// Constants section
consts := gen.parser.list_constants(module_name)
if consts.len > 0 {
md += gen.constants_section(consts)
}
// Structs section
structs := gen.parser.list_structs(module_name)
if structs.len > 0 {
md += gen.structs_section(structs, module_name)
}
// Functions section
functions := gen.parser.list_functions(module_name)
if functions.len > 0 {
md += gen.functions_section(functions, module_name)
}
// Interfaces section
interfaces := gen.parser.list_interfaces(module_name)
if interfaces.len > 0 {
md += gen.interfaces_section(interfaces)
}
return md
}
// imports_section generates imports documentation
fn (gen CodeGenerator) imports_section(imports []code.Import) string {
mut md := '## Imports\n\n'
for imp in imports {
md += '- `' + imp.mod + '`\n'
}
md += '\n'
return md
}
// constants_section generates constants documentation
fn (gen CodeGenerator) constants_section(consts []code.Const) string {
mut md := '## Constants\n\n'
for const_ in consts {
md += '- `' + const_.name + '` = `' + const_.value + '`\n'
}
md += '\n'
return md
}
// structs_section generates structs documentation
fn (gen CodeGenerator) structs_section(structs []code.Struct, module_name string) string {
mut md := '## Structs\n\n'
for struct_ in structs {
md += gen.struct_to_markdown(struct_)
}
return md
}
// functions_section generates functions documentation
fn (gen CodeGenerator) functions_section(functions []code.Function, module_name string) string {
mut md := '## Functions & Methods\n\n'
// Separate regular functions and methods
regular_functions := functions.filter(it.receiver.typ.symbol() == '')
methods := functions.filter(it.receiver.typ.symbol() != '')
// Regular functions
if regular_functions.len > 0 {
md += '### Functions\n\n'
for func in regular_functions {
md += gen.function_to_markdown(func)
}
}
// Methods (grouped by struct)
if methods.len > 0 {
md += '### Methods\n\n'
structs := gen.parser.list_structs(module_name)
for struct_ in structs {
struct_methods := methods.filter(it.receiver.typ.symbol().contains(struct_.name))
if struct_methods.len > 0 {
md += '#### ' + struct_.name + '\n\n'
for method in struct_methods {
md += gen.function_to_markdown(method)
}
}
}
}
return md
}
// interfaces_section generates interfaces documentation
fn (gen CodeGenerator) interfaces_section(interfaces []code.Interface) string {
mut md := '## Interfaces\n\n'
for iface in interfaces {
md += '### ' + iface.name + '\n\n'
if iface.description != '' {
md += iface.description + '\n\n'
}
md += '```v\n'
if iface.is_pub {
md += 'pub '
}
md += 'interface ' + iface.name + ' {\n'
for field in iface.fields {
md += ' ' + field.name + ': ' + field.typ.symbol() + '\n'
}
md += '}\n```\n\n'
}
return md
}
// struct_to_markdown converts struct to markdown
fn (gen CodeGenerator) struct_to_markdown(struct_ code.Struct) string {
mut md := '### '
if struct_.is_pub {
md += '**pub** '
}
md += 'struct ' + struct_.name + '\n\n'
if struct_.description != '' {
md += struct_.description + '\n\n'
}
md += '```v\n'
if struct_.is_pub {
md += 'pub '
}
md += 'struct ' + struct_.name + ' {\n'
for field in struct_.fields {
md += ' ' + field.name + ' ' + field.typ.symbol() + '\n'
}
md += '}\n'
md += '```\n\n'
// Field documentation
if struct_.fields.len > 0 {
md += '**Fields:**\n\n'
for field in struct_.fields {
visibility := if field.is_pub { 'public' } else { 'private' }
mutability := if field.is_mut { ', mutable' } else { '' }
md += '- `' + field.name + '` (`' + field.typ.symbol() + '`)' + mutability + ' - ' +
visibility + '\n'
if field.description != '' {
md += ' - ' + field.description + '\n'
}
}
md += '\n'
}
return md
}
// function_to_markdown converts function to markdown
fn (gen CodeGenerator) function_to_markdown(func code.Function) string {
mut md := ''
// Function signature
signature := gen.function_signature(func)
md += '- `' + signature + '`\n'
// Description
if func.description != '' {
md += ' - *' + func.description + '*\n'
}
// Parameters
if func.params.len > 0 {
md += '\n **Parameters:**\n'
for param in func.params {
md += ' - `' + param.name + '` (`' + param.typ.symbol() + '`)'
if param.description != '' {
md += ' - ' + param.description
}
md += '\n'
}
}
// Return type
if func.result.typ.symbol() != '' {
md += '\n **Returns:** `' + func.result.typ.symbol() + '`\n'
}
md += '\n'
return md
}
// function_signature generates a function signature string
fn (gen CodeGenerator) function_signature(func code.Function) string {
mut sig := if func.is_pub { 'pub ' } else { '' }
if func.receiver.name != '' {
sig += '(' + func.receiver.name + ' ' + func.receiver.typ.symbol() + ') '
}
sig += func.name
// Parameters
params := func.params.map(it.name + ': ' + it.typ.symbol()).join(', ')
sig += '(' + params + ')'
// Return type
if func.result.typ.symbol() != '' {
sig += ' -> ' + func.result.typ.symbol()
}
return sig
}
// module_to_filename converts module name to filename
// e.g., incubaid.herolib.core.code -> code__core__code.md
pub fn (gen CodeGenerator) module_to_filename(module_name string) string {
// Get last part after last dot, then add __ and rest in reverse
parts := module_name.split('.')
filename := parts[parts.len - 1]
return filename + '.md'
}

27
lib/core/codegenerator/factory.v Normal file
View File

@@ -0,0 +1,27 @@
module codegenerator
import incubaid.herolib.core.codeparser
@[params]
pub struct GeneratorOptions {
pub:
parser_path string @[required]
output_dir string @[required]
recursive bool = true
format bool = true
}
pub fn new(args GeneratorOptions) !CodeGenerator {
mut parser := codeparser.new(
path: args.parser_path
recursive: args.recursive
)!
parser.parse()!
return CodeGenerator{
parser: parser
output_dir: args.output_dir
format: args.format
}
}

31
lib/core/codegenerator/markdown_gen.v Normal file
View File

@@ -0,0 +1,31 @@
module codegenerator
import incubaid.herolib.core.pathlib
pub struct MarkdownGenerator {
pub mut:
generator CodeGenerator
output_dir string
}
// write_all writes all generated markdown files to disk
pub fn (mut mgen MarkdownGenerator) write_all() ! {
modules := mgen.generator.parser.list_modules()
// Ensure output directory exists
mut out_dir := pathlib.get_dir(path: mgen.output_dir, create: true)!
for module_name in modules {
mgen.write_module(module_name)!
}
}
// write_module writes a single module's markdown to disk
pub fn (mut mgen MarkdownGenerator) write_module(module_name string) ! {
md := mgen.generator.module_to_markdown(module_name)!
filename := mgen.generator.module_to_filename(module_name)
filepath := mgen.output_dir + '/' + filename
mut file := pathlib.get_file(path: filepath, create: true)!
file.write(md)!
}

188
lib/core/codegenerator/markdown_test.v Normal file
View File

@@ -0,0 +1,188 @@
module codegenerator
import incubaid.herolib.ui.console
import incubaid.herolib.core.codeparser
import incubaid.herolib.core.pathlib
import os
fn test_markdown_generation() {
console.print_header('CodeGenerator Markdown Test')
console.print_lf(1)
// Setup: Use the same test data as codeparser
test_dir := setup_test_directory()
defer {
os.rmdir_all(test_dir) or {}
}
// Create output directory
output_dir := '/tmp/codegen_output'
os.rmdir_all(output_dir) or {}
os.mkdir_all(output_dir) or { panic('Failed to create output dir') }
defer {
os.rmdir_all(output_dir) or {}
}
// Create generator
console.print_item('Creating CodeGenerator...')
mut gen := new(
parser_path: test_dir
output_dir: output_dir
recursive: true
)!
console.print_item('Parser found ${gen.parser.list_modules().len} modules')
console.print_lf(1)
// Test filename conversion
console.print_header('Test 1: Filename Conversion')
struct TestCase {
module_name string
expected string
}
test_cases := [
TestCase{
module_name: 'incubaid.herolib.core.code'
expected: 'code.md'
},
TestCase{
module_name: 'testdata'
expected: 'testdata.md'
},
TestCase{
module_name: 'testdata.services'
expected: 'services.md'
},
]
for test_case in test_cases {
result := gen.module_to_filename(test_case.module_name)
assert result == test_case.expected, 'Expected ${test_case.expected}, got ${result}'
console.print_item(' ✓ ${test_case.module_name} -> ${result}')
}
console.print_lf(1)
// Test module documentation generation
console.print_header('Test 2: Module Documentation Generation')
// Get a testdata module
modules := gen.parser.list_modules()
testdata_modules := modules.filter(it.contains('testdata'))
assert testdata_modules.len > 0, 'No testdata modules found'
for mod_name in testdata_modules {
console.print_item('Generating docs for: ${mod_name}')
md := gen.module_to_markdown(mod_name)!
// Validate markdown content
assert md.len > 0, 'Generated markdown is empty'
assert md.contains('# Module:'), 'Missing module header'
// List basic structure checks
structs := gen.parser.list_structs(mod_name)
functions := gen.parser.list_functions(mod_name)
consts := gen.parser.list_constants(mod_name)
if structs.len > 0 {
assert md.contains('## Structs'), 'Missing Structs section'
console.print_item(' - Found ${structs.len} structs')
}
if functions.len > 0 {
assert md.contains('## Functions'), 'Missing Functions section'
console.print_item(' - Found ${functions.len} functions')
}
if consts.len > 0 {
assert md.contains('## Constants'), 'Missing Constants section'
console.print_item(' - Found ${consts.len} constants')
}
}
console.print_lf(1)
// Test file writing
console.print_header('Test 3: Write Generated Files')
for mod_name in testdata_modules {
gen.generate_module(mod_name)!
}
// Verify files were created
files := os.ls(output_dir)!
assert files.len > 0, 'No files generated'
console.print_item('Generated ${files.len} markdown files:')
for file in files {
console.print_item(' - ${file}')
// Verify file content
filepath := os.join_path(output_dir, file)
content := os.read_file(filepath)!
assert content.len > 0, 'Generated file is empty: ${file}'
}
console.print_lf(1)
// Test content validation
console.print_header('Test 4: Content Validation')
for file in files {
filepath := os.join_path(output_dir, file)
content := os.read_file(filepath)!
// Check for required sections
has_module_header := content.contains('# Module:')
has_imports := content.contains('## Imports') || !content.contains('import ')
has_valid_format := content.contains('```v')
assert has_module_header, '${file}: Missing module header'
assert has_valid_format || file.contains('services'), '${file}: Invalid markdown format'
console.print_item(' ✓ ${file}: Valid content')
}
console.print_lf(1)
console.print_green('✓ All CodeGenerator tests passed!')
}
// Helper: Setup test directory (copy from codeparser test)
fn setup_test_directory() string {
test_dir := '/tmp/codegen_test_data'
os.rmdir_all(test_dir) or {}
current_file := @FILE
current_dir := os.dir(current_file)
// Navigate to codeparser testdata
codeparser_dir := os.join_path(os.dir(current_dir), 'codeparser')
testdata_dir := os.join_path(codeparser_dir, 'testdata')
if !os.is_dir(testdata_dir) {
panic('testdata directory not found at: ${testdata_dir}')
}
os.mkdir_all(test_dir) or { panic('Failed to create test directory') }
copy_directory(testdata_dir, test_dir) or { panic('Failed to copy testdata: ${err}') }
return test_dir
}
fn copy_directory(src string, dst string) ! {
entries := os.ls(src)!
for entry in entries {
src_path := os.join_path(src, entry)
dst_path := os.join_path(dst, entry)
if os.is_dir(src_path) {
os.mkdir_all(dst_path)!
copy_directory(src_path, dst_path)!
} else {
content := os.read_file(src_path)!
os.write_file(dst_path, content)!
}
}
}

1
lib/core/codegenerator/templates/function.md.template Normal file
View File

@@ -0,0 +1 @@
fn ${func.name}(${func.params.map(it.name + ': ' + it.typ.symbol()).join(', ')}) ${func.result.typ.symbol()}

0
lib/core/code/templates/comment/comment.py → lib/core/codegenerator/templates/imports.md.template
View File

5
lib/core/codegenerator/templates/module.md.template Normal file
View File

@@ -0,0 +1,5 @@
# Module: ${module_name}
This module provides functionality for code generation and documentation.
**Location:** `${module_name.replace('.', '/')}`

2
lib/core/codegenerator/templates/struct.md.template Normal file
View File

@@ -0,0 +1,2 @@
struct ${struct_.name} {
}

124
lib/core/codeparser/README.md Normal file
View File

@@ -0,0 +1,124 @@
# CodeParser Module
The `codeparser` module provides a comprehensive indexing and analysis system for V codebases. It walks directory trees, parses all V files, and allows efficient searching, filtering, and analysis of code structures.
## Features
- **Directory Scanning**: Automatically walks directory trees and finds all V files
- **Batch Parsing**: Parses multiple files efficiently
- **Indexing**: Indexes code by module, structs, functions, interfaces, constants
- **Search**: Find specific items by name
- **Filtering**: Use predicates to filter code items
- **Statistics**: Get module statistics (file count, struct count, etc.)
- **Export**: Export complete codebase structure as JSON
- **Error Handling**: Gracefully handles parse errors
## Basic Usage
```v
import incubaid.herolib.core.codeparser
// Create a parser for a directory
mut parser := codeparser.new('/path/to/herolib')!
// List all modules
modules := parser.list_modules()
for mod in modules {
println('Module: ${mod}')
}
// Find a specific struct
struct_ := parser.find_struct('User', 'mymodule')!
println('Struct: ${struct_.name}')
// List all public functions
pub_fns := parser.filter_public_functions()
// Get methods on a struct
methods := parser.list_methods_on_struct('User')
// Export to JSON
json_str := parser.to_json()!
```
## API Reference
### Factory
- `new(root_dir: string) !CodeParser` - Create parser for a directory
### Listers
- `list_modules() []string` - All modules
- `list_files() []string` - All files
- `list_files_in_module(module: string) []string` - Files in module
- `list_structs(module: string = '') []Struct` - All structs
- `list_functions(module: string = '') []Function` - All functions
- `list_interfaces(module: string = '') []Interface` - All interfaces
- `list_methods_on_struct(struct: string, module: string = '') []Function` - Methods
- `list_imports(module: string = '') []Import` - All imports
- `list_constants(module: string = '') []Const` - All constants
### Finders
- `find_struct(name: string, module: string = '') !Struct`
- `find_function(name: string, module: string = '') !Function`
- `find_interface(name: string, module: string = '') !Interface`
- `find_method(struct: string, method: string, module: string = '') !Function`
- `find_module(name: string) !ParsedModule`
- `find_file(path: string) !ParsedFile`
- `find_structs_with_method(method: string, module: string = '') []string`
- `find_callers(function: string, module: string = '') []Function`
### Filters
- `filter_structs(predicate: fn(Struct) bool, module: string = '') []Struct`
- `filter_functions(predicate: fn(Function) bool, module: string = '') []Function`
- `filter_public_structs(module: string = '') []Struct`
- `filter_public_functions(module: string = '') []Function`
- `filter_functions_with_receiver(module: string = '') []Function`
- `filter_functions_returning_error(module: string = '') []Function`
- `filter_structs_with_field(type: string, module: string = '') []Struct`
- `filter_structs_by_name(pattern: string, module: string = '') []Struct`
- `filter_functions_by_name(pattern: string, module: string = '') []Function`
### Export
- `to_json(module: string = '') !string` - Export to JSON
- `to_json_pretty(module: string = '') !string` - Pretty-printed JSON
### Error Handling
- `has_errors() bool` - Check if parsing errors occurred
- `error_count() int` - Get number of errors
- `print_errors()` - Print all errors
## Example: Analyzing a Module
```v
import incubaid.herolib.core.codeparser
mut parser := codeparser.new(os.home_dir() + '/code/github/incubaid/herolib/lib/core')!
// Get all public functions in the 'pathlib' module
pub_fns := parser.filter_public_functions('incubaid.herolib.core.pathlib')
for fn in pub_fns {
println('${fn.name}() -> ${fn.result.typ.symbol()}')
}
// Find all structs with a specific method
structs := parser.find_structs_with_method('read')
// Export pathlib module to JSON
json_str := parser.to_json('incubaid.herolib.core.pathlib')!
println(json_str)
```
## Implementation Notes
1. **Lazy Parsing**: Files are parsed only when needed
2. **Error Recovery**: Parsing errors don't stop the indexing process
3. **Memory Efficient**: Maintains index in memory but doesn't duplicate code
4. **Module Agnostic**: Works with any V module structure
5. **Cross-Module Search**: Can search across entire codebase or single module

363
lib/core/codeparser/advanced_test.v Normal file
View File

@@ -0,0 +1,363 @@
module codeparser
import incubaid.herolib.ui.console
import incubaid.herolib.core.pathlib
import incubaid.herolib.core.code
import os
fn test_comprehensive_code_parsing() {
console.print_header('Comprehensive Code Parsing Tests')
console.print_lf(1)
// Setup test files by copying testdata
test_dir := setup_test_directory()
console.print_item('Copied testdata to: ${test_dir}')
console.print_lf(1)
// Run all tests
test_module_parsing()
test_struct_parsing()
test_function_parsing()
test_imports_and_modules()
test_type_system()
test_visibility_modifiers()
test_method_parsing()
test_constants_parsing()
console.print_green('✓ All comprehensive tests passed!')
console.print_lf(1)
// Cleanup
os.rmdir_all(test_dir) or {}
console.print_item('Cleaned up test directory')
}
// setup_test_directory copies the testdata directory to /tmp/codeparsertest
fn setup_test_directory() string {
test_dir := '/tmp/codeparsertest'
// Remove existing test directory
os.rmdir_all(test_dir) or {}
// Find the testdata directory relative to this file
current_file := @FILE
current_dir := os.dir(current_file)
testdata_dir := os.join_path(current_dir, 'testdata')
// Verify testdata directory exists
if !os.is_dir(testdata_dir) {
panic('testdata directory not found at: ${testdata_dir}')
}
// Copy testdata to test directory
os.mkdir_all(test_dir) or { panic('Failed to create test directory') }
copy_directory(testdata_dir, test_dir) or { panic('Failed to copy testdata: ${err}') }
return test_dir
}
// copy_directory recursively copies a directory and all its contents
fn copy_directory(src string, dst string) ! {
entries := os.ls(src)!
for entry in entries {
src_path := os.join_path(src, entry)
dst_path := os.join_path(dst, entry)
if os.is_dir(src_path) {
os.mkdir_all(dst_path)!
copy_directory(src_path, dst_path)!
} else {
content := os.read_file(src_path)!
os.write_file(dst_path, content)!
}
}
}
fn test_module_parsing() {
console.print_header('Test 1: Module and File Parsing')
mut myparser := new(path: '/tmp/codeparsertest', recursive: true)!
myparser.parse()!
v_files := myparser.list_files()
console.print_item('Found ${v_files.len} V files')
mut total_items := 0
for file_path in v_files {
if parsed_file := myparser.parsed_files[file_path] {
console.print_item(' ✓ ${os.base(file_path)}: ${parsed_file.vfile.items.len} items')
total_items += parsed_file.vfile.items.len
}
}
assert v_files.len >= 7, 'Expected at least 7 V files, got ${v_files.len}'
assert total_items > 0, 'Expected to parse some items'
console.print_green('✓ Module parsing test passed')
console.print_lf(1)
}
fn test_struct_parsing() {
console.print_header('Test 2: Struct Parsing')
models_file := os.join_path('/tmp/codeparsertest', 'models.v')
content := os.read_file(models_file) or {
assert false, 'Failed to read models.v'
return
}
vfile := code.parse_vfile(content) or {
assert false, 'Failed to parse models.v: ${err}'
return
}
structs := vfile.structs()
assert structs.len >= 3, 'Expected at least 3 structs, got ${structs.len}'
// Check User struct
user_struct := structs.filter(it.name == 'User')
assert user_struct.len == 1, 'User struct not found'
user := user_struct[0]
assert user.is_pub == true, 'User struct should be public'
assert user.fields.len == 6, 'User struct should have 6 fields, got ${user.fields.len}'
console.print_item(' ✓ User struct: ${user.fields.len} fields (public)')
// Check Profile struct
profile_struct := structs.filter(it.name == 'Profile')
assert profile_struct.len == 1, 'Profile struct not found'
assert profile_struct[0].is_pub == true, 'Profile should be public'
console.print_item(' ✓ Profile struct: ${profile_struct[0].fields.len} fields (public)')
// Check Settings struct (private)
settings_struct := structs.filter(it.name == 'Settings')
assert settings_struct.len == 1, 'Settings struct not found'
assert settings_struct[0].is_pub == false, 'Settings should be private'
console.print_item(' ✓ Settings struct: ${settings_struct[0].fields.len} fields (private)')
// Check InternalConfig struct
config_struct := structs.filter(it.name == 'InternalConfig')
assert config_struct.len == 1, 'InternalConfig struct not found'
assert config_struct[0].is_pub == false, 'InternalConfig should be private'
console.print_item(' ✓ InternalConfig struct (private)')
console.print_green('✓ Struct parsing test passed')
console.print_lf(1)
}
fn test_function_parsing() {
console.print_header('Test 3: Function Parsing')
mut myparser := new(path: '/tmp/codeparsertest', recursive: true)!
myparser.parse()!
mut functions := []code.Function{}
for _, parsed_file in myparser.parsed_files {
functions << parsed_file.vfile.functions()
}
pub_functions := functions.filter(it.is_pub)
priv_functions := functions.filter(!it.is_pub)
assert pub_functions.len >= 8, 'Expected at least 8 public functions, got ${pub_functions.len}'
assert priv_functions.len >= 4, 'Expected at least 4 private functions, got ${priv_functions.len}'
// Check create_user function
create_user_fn := functions.filter(it.name == 'create_user')
assert create_user_fn.len == 1, 'create_user function not found'
create_fn := create_user_fn[0]
assert create_fn.is_pub == true, 'create_user should be public'
assert create_fn.params.len == 2, 'create_user should have 2 parameters'
console.print_item(' ✓ create_user: ${create_fn.params.len} params, public')
// Check get_user function
get_user_fn := functions.filter(it.name == 'get_user')
assert get_user_fn.len == 1, 'get_user function not found'
assert get_user_fn[0].is_pub == true
console.print_item(' ✓ get_user: public function')
// Check delete_user function
delete_user_fn := functions.filter(it.name == 'delete_user')
assert delete_user_fn.len == 1, 'delete_user function not found'
console.print_item(' ✓ delete_user: public function')
// Check validate_email (private)
validate_fn := functions.filter(it.name == 'validate_email')
assert validate_fn.len == 1, 'validate_email function not found'
assert validate_fn[0].is_pub == false, 'validate_email should be private'
console.print_item(' ✓ validate_email: private function')
console.print_green('✓ Function parsing test passed')
console.print_lf(1)
}
fn test_imports_and_modules() {
console.print_header('Test 4: Imports and Module Names')
models_file := os.join_path('/tmp/codeparsertest', 'models.v')
content := os.read_file(models_file) or {
assert false, 'Failed to read models.v'
return
}
vfile := code.parse_vfile(content) or {
assert false, 'Failed to parse models.v: ${err}'
return
}
assert vfile.mod == 'testdata', 'Module name should be testdata, got ${vfile.mod}'
assert vfile.imports.len == 2, 'Expected 2 imports, got ${vfile.imports.len}'
console.print_item(' ✓ Module name: ${vfile.mod}')
console.print_item(' ✓ Imports: ${vfile.imports.len}')
for import_ in vfile.imports {
console.print_item(' - ${import_.mod}')
}
assert 'time' in vfile.imports.map(it.mod), 'time import not found'
assert 'os' in vfile.imports.map(it.mod), 'os import not found'
console.print_green('✓ Import and module test passed')
console.print_lf(1)
}
fn test_type_system() {
console.print_header('Test 5: Type System')
models_file := os.join_path('/tmp/codeparsertest', 'models.v')
content := os.read_file(models_file) or {
assert false, 'Failed to read models.v'
return
}
vfile := code.parse_vfile(content) or {
assert false, 'Failed to parse models.v: ${err}'
return
}
structs := vfile.structs()
user_struct := structs.filter(it.name == 'User')[0]
// Test different field types
id_field := user_struct.fields.filter(it.name == 'id')[0]
assert id_field.typ.symbol() == 'int', 'id field should be int, got ${id_field.typ.symbol()}'
email_field := user_struct.fields.filter(it.name == 'email')[0]
assert email_field.typ.symbol() == 'string', 'email field should be string'
active_field := user_struct.fields.filter(it.name == 'active')[0]
assert active_field.typ.symbol() == 'bool', 'active field should be bool'
console.print_item(' ✓ Integer type: ${id_field.typ.symbol()}')
console.print_item(' ✓ String type: ${email_field.typ.symbol()}')
console.print_item(' ✓ Boolean type: ${active_field.typ.symbol()}')
console.print_green('✓ Type system test passed')
console.print_lf(1)
}
fn test_visibility_modifiers() {
console.print_header('Test 6: Visibility Modifiers')
models_file := os.join_path('/tmp/codeparsertest', 'models.v')
content := os.read_file(models_file) or {
assert false, 'Failed to read models.v'
return
}
vfile := code.parse_vfile(content) or {
assert false, 'Failed to parse models.v: ${err}'
return
}
structs := vfile.structs()
// Check User struct visibility
user_struct := structs.filter(it.name == 'User')[0]
assert user_struct.is_pub == true, 'User struct should be public'
pub_fields := user_struct.fields.filter(it.is_pub)
mut_fields := user_struct.fields.filter(it.is_mut)
console.print_item(' ✓ User struct: public')
console.print_item(' - Public fields: ${pub_fields.len}')
console.print_item(' - Mutable fields: ${mut_fields.len}')
// Check InternalConfig visibility
config_struct := structs.filter(it.name == 'InternalConfig')[0]
assert config_struct.is_pub == false, 'InternalConfig should be private'
console.print_item(' ✓ InternalConfig: private')
console.print_green('✓ Visibility modifiers test passed')
console.print_lf(1)
}
fn test_method_parsing() {
console.print_header('Test 7: Method Parsing')
mut myparser := new(path: '/tmp/codeparsertest', recursive: true)!
myparser.parse()!
mut methods := []code.Function{}
for _, parsed_file in myparser.parsed_files {
methods << parsed_file.vfile.functions().filter(it.receiver.name != '')
}
assert methods.len >= 11, 'Expected at least 11 methods, got ${methods.len}'
// Check activate method
activate_methods := methods.filter(it.name == 'activate')
assert activate_methods.len == 1, 'activate method not found'
assert activate_methods[0].receiver.mutable == true, 'activate should have mutable receiver'
console.print_item(' ✓ activate: mutable method')
// Check is_active method
is_active_methods := methods.filter(it.name == 'is_active')
assert is_active_methods.len == 1, 'is_active method not found'
assert is_active_methods[0].receiver.mutable == false, 'is_active should have immutable receiver'
console.print_item(' ✓ is_active: immutable method')
// Check get_display_name method
display_methods := methods.filter(it.name == 'get_display_name')
assert display_methods.len == 1, 'get_display_name method not found'
console.print_item(' ✓ get_display_name: method found')
console.print_green('✓ Method parsing test passed')
console.print_lf(1)
}
fn test_constants_parsing() {
console.print_header('Test 8: Constants Parsing')
models_file := os.join_path('/tmp/codeparsertest', 'models.v')
content := os.read_file(models_file) or {
assert false, 'Failed to read models.v'
return
}
vfile := code.parse_vfile(content) or {
assert false, 'Failed to parse models.v: ${err}'
return
}
assert vfile.consts.len == 3, 'Expected 3 constants, got ${vfile.consts.len}'
// Check app_version constant
version_const := vfile.consts.filter(it.name == 'app_version')
assert version_const.len == 1, 'app_version constant not found'
console.print_item(' ✓ app_version: ${version_const[0].value}')
// Check max_users constant
max_users_const := vfile.consts.filter(it.name == 'max_users')
assert max_users_const.len == 1, 'max_users constant not found'
console.print_item(' ✓ max_users: ${max_users_const[0].value}')
// Check default_timeout constant
timeout_const := vfile.consts.filter(it.name == 'default_timeout')
assert timeout_const.len == 1, 'default_timeout constant not found'
console.print_item(' ✓ default_timeout: ${timeout_const[0].value}')
console.print_green('✓ Constants parsing test passed')
console.print_lf(1)
}

147
lib/core/codeparser/codeparser.v Normal file
View File

@@ -0,0 +1,147 @@
module codeparser
import incubaid.herolib.core.code
import incubaid.herolib.core.pathlib
// ParseError represents an error that occurred while parsing a file
pub struct ParseError {
pub:
file_path string
error string
}
// ParsedFile represents a successfully parsed V file
pub struct ParsedFile {
pub:
path string
module_name string
vfile code.VFile
}
pub struct ModuleStats {
pub mut:
file_count int
struct_count int
function_count int
interface_count int
const_count int
}
pub struct ParsedModule {
pub:
name string
file_paths []string
stats ModuleStats
}
pub struct CodeParser {
pub mut:
root_dir string
options ParserOptions
parsed_files map[string]ParsedFile
modules map[string][]string
parse_errors []ParseError
}
// scan_directory recursively walks the directory and identifies all V files
// Files are stored but not parsed until parse() is called
fn (mut parser CodeParser) scan_directory() ! {
mut root := pathlib.get_dir(path: parser.root_dir, create: false)!
if !root.exists() {
return error('root directory does not exist: ${parser.root_dir}')
}
// Use pathlib's recursive listing capability
mut items := root.list(recursive: parser.options.recursive)!
for item in items.paths {
// Skip non-V files
if !item.path.ends_with('.v') {
continue
}
// Skip generated files (ending with _.v)
if item.path.ends_with('_.v') {
continue
}
// Check exclude patterns
should_skip := parser.options.exclude_patterns.any(item.path.contains(it))
if should_skip {
continue
}
// Store file path for lazy parsing
parsed_file := ParsedFile{
path: item.path
module_name: ''
vfile: code.VFile{}
}
parser.parsed_files[item.path] = parsed_file
}
}
// parse processes all V files that were scanned and parses them
pub fn (mut parser CodeParser) parse() ! {
for file_path, _ in parser.parsed_files {
if parser.parsed_files[file_path].vfile.mod == '' {
// Only parse if not already parsed
parser.parse_file(file_path)!
}
}
}
// parse_file parses a single V file and adds it to the index
pub fn (mut parser CodeParser) parse_file(file_path string) ! {
mut file := pathlib.get_file(path: file_path) or {
parser.parse_errors << ParseError{
file_path: file_path
error: 'Failed to access file: ${err.msg()}'
}
return error('Failed to access file: ${err.msg()}')
}
content := file.read() or {
parser.parse_errors << ParseError{
file_path: file_path
error: 'Failed to read file: ${err.msg()}'
}
return error('Failed to read file: ${err.msg()}')
}
// Parse the V file
vfile := code.parse_vfile(content) or {
parser.parse_errors << ParseError{
file_path: file_path
error: 'Parse error: ${err.msg()}'
}
return error('Parse error: ${err.msg()}')
}
parsed_file := ParsedFile{
path: file_path
module_name: vfile.mod
vfile: vfile
}
parser.parsed_files[file_path] = parsed_file
// Index by module
if vfile.mod !in parser.modules {
parser.modules[vfile.mod] = []string{}
}
if file_path !in parser.modules[vfile.mod] {
parser.modules[vfile.mod] << file_path
}
}
// has_errors returns true if any parsing errors occurred
pub fn (parser CodeParser) has_errors() bool {
return parser.parse_errors.len > 0
}
// error_count returns the number of parsing errors
pub fn (parser CodeParser) error_count() int {
return parser.parse_errors.len
}

26
lib/core/codeparser/factory.v Normal file
View File

@@ -0,0 +1,26 @@
module codeparser
// import incubaid.herolib.core.pathlib
// import incubaid.herolib.core.code
@[params]
pub struct ParserOptions {
pub:
path string @[required]
recursive bool = true
exclude_patterns []string
include_patterns []string = ['*.v']
}
// new creates a CodeParser and scans the given root directory
pub fn new(args ParserOptions) !CodeParser {
mut parser := CodeParser{
root_dir: args.path
options: args
parsed_files: map[string]ParsedFile{}
modules: map[string][]string{}
parse_errors: []ParseError{}
}
parser.scan_directory()!
return parser
}

84
lib/core/codeparser/filters.v Normal file
View File

@@ -0,0 +1,84 @@
module codeparser
import incubaid.herolib.core.code
import regex
@[params]
pub struct FilterOptions {
pub:
module_name string
name_filter string // just partial match
is_public bool
has_receiver bool
}
// structs returns a filtered list of all structs found in the parsed files
pub fn (parser CodeParser) structs(options FilterOptions) []code.Struct {
mut result := []code.Struct{}
for _, file in parser.parsed_files {
if options.module_name != '' && file.module_name != options.module_name {
continue
}
for struct_ in file.vfile.structs() {
if options.name_filter.len > 0 {
if !struct_.name.contains(options.name_filter) {
continue
}
}
if options.is_public && !struct_.is_pub {
continue
}
result << struct_
}
}
return result
}
// functions returns a filtered list of all functions found in the parsed files
pub fn (parser CodeParser) functions(options FilterOptions) []code.Function {
mut result := []code.Function{}
for _, file in parser.parsed_files {
if options.module_name != '' && file.module_name != options.module_name {
continue
}
for func in file.vfile.functions() {
if options.name_filter.len > 0 {
if !func.name.contains(options.name_filter) {
continue
}
}
if options.is_public && !func.is_pub {
continue
}
if options.has_receiver && func.receiver.typ.symbol() == '' {
continue
}
result << func
}
}
return result
}
// filter_public_structs returns all public structs
pub fn (parser CodeParser) filter_public_structs(module_name string) []code.Struct {
return parser.structs(
module_name: module_name
is_public: true
)
}
// filter_public_functions returns all public functions
pub fn (parser CodeParser) filter_public_functions(module_name string) []code.Function {
return parser.functions(
module_name: module_name
is_public: true
)
}
// filter_methods returns all functions with receivers (methods)
pub fn (parser CodeParser) filter_methods(module_name string) []code.Function {
return parser.functions(
module_name: module_name
has_receiver: true
)
}

137
lib/core/codeparser/finders.v Normal file
View File

@@ -0,0 +1,137 @@
module codeparser
import incubaid.herolib.core.code
@[params]
pub struct FinderOptions {
pub:
name string @[required]
struct_name string // only useful for methods on structs
module_name string
}
// find_struct searches for a struct by name
pub fn (parser CodeParser) find_struct(args FinderOptions) !code.Struct {
for _, parsed_file in parser.parsed_files {
if args.module_name != '' && parsed_file.module_name != args.module_name {
continue
}
structs := parsed_file.vfile.structs()
for struct_ in structs {
if struct_.name == args.name {
return struct_
}
}
}
module_suffix := if args.module_name != '' { ' in module \'${args.module_name}\'' } else { '' }
return error('struct \'${args.name}\' not found${module_suffix}')
}
// find_function searches for a function by name
pub fn (parser CodeParser) find_function(args FinderOptions) !code.Function {
for _, parsed_file in parser.parsed_files {
if args.module_name != '' && parsed_file.module_name != args.module_name {
continue
}
if func := parsed_file.vfile.get_function(args.name) {
return func
}
}
module_suffix := if args.module_name != '' { ' in module \'${args.module_name}\'' } else { '' }
return error('function \'${args.name}\' not found${module_suffix}')
}
// find_interface searches for an interface by name
pub fn (parser CodeParser) find_interface(args FinderOptions) !code.Interface {
for _, parsed_file in parser.parsed_files {
if args.module_name != '' && parsed_file.module_name != args.module_name {
continue
}
for item in parsed_file.vfile.items {
if item is code.Interface {
iface := item as code.Interface
if iface.name == args.name {
return iface
}
}
}
}
module_suffix := if args.module_name != '' { ' in module \'${args.module_name}\'' } else { '' }
return error('interface \'${args.name}\' not found${module_suffix}')
}
// find_method searches for a method on a struct
pub fn (parser CodeParser) find_method(args FinderOptions) !code.Function {
methods := parser.list_methods_on_struct(args.struct_name, args.module_name)
for method in methods {
if method.name == args.name {
return method
}
}
module_suffix := if args.module_name != '' { ' in module \'${args.module_name}\'' } else { '' }
return error('method \'${args.name}\' on struct \'${args.struct_name}\' not found${module_suffix}')
}
// find_module searches for a module by name
pub fn (parser CodeParser) find_module(module_name string) !ParsedModule {
if module_name !in parser.modules {
return error('module \'${module_name}\' not found')
}
file_paths := parser.modules[module_name]
stats := parser.get_module_stats(module_name)
return ParsedModule{
name: module_name
file_paths: file_paths
stats: stats
}
}
// find_file retrieves parsed file information
pub fn (parser CodeParser) find_file(path string) !ParsedFile {
if path !in parser.parsed_files {
return error('file \'${path}\' not found in parsed files')
}
return parser.parsed_files[path]
}
// find_structs_with_method finds all structs that have a specific method
pub fn (parser CodeParser) find_structs_with_method(args FinderOptions) []string {
mut struct_names := []string{}
functions := parser.list_functions(args.module_name)
for func in functions {
if func.name == args.name && func.receiver.name != '' {
struct_type := func.receiver.typ.symbol()
if struct_type !in struct_names {
struct_names << struct_type
}
}
}
return struct_names
}
// find_callers finds all functions that call a specific function (basic text matching)
pub fn (parser CodeParser) find_callers(args FinderOptions) []code.Function {
mut callers := []code.Function{}
functions := parser.list_functions(args.module_name)
for func in functions {
if func.body.contains(args.name) {
callers << func
}
}
return callers
}

89
lib/core/codeparser/getters.v Normal file
View File

@@ -0,0 +1,89 @@
module codeparser
import incubaid.herolib.core.code
// get_module_stats calculates statistics for a module
pub fn (parser CodeParser) get_module_stats(module_name string) ModuleStats {
mut stats := ModuleStats{}
file_paths := parser.modules[module_name] or { []string{} }
for file_path in file_paths {
if parsed_file := parser.parsed_files[file_path] {
stats.file_count++
stats.struct_count += parsed_file.vfile.structs().len
stats.function_count += parsed_file.vfile.functions().len
for item in parsed_file.vfile.items {
if item is code.Interface {
stats.interface_count++
}
}
stats.const_count += parsed_file.vfile.consts.len
}
}
return stats
}
// get_parsed_file returns the parsed file for a given path
pub fn (parser CodeParser) get_parsed_file(file_path string) ?ParsedFile {
return parser.parsed_files[file_path]
}
// all_structs returns all structs from all parsed files
pub fn (p CodeParser) all_structs() []code.Struct {
mut all := []code.Struct{}
for _, file in p.parsed_files {
all << file.vfile.structs()
}
return all
}
// all_functions returns all functions from all parsed files
pub fn (p CodeParser) all_functions() []code.Function {
mut all := []code.Function{}
for _, file in p.parsed_files {
all << file.vfile.functions()
}
return all
}
// all_consts returns all constants from all parsed files
pub fn (p CodeParser) all_consts() []code.Const {
mut all := []code.Const{}
for _, file in p.parsed_files {
all << file.vfile.consts
}
return all
}
// all_imports returns a map of all unique imports
pub fn (p CodeParser) all_imports() map[string]bool {
mut all := map[string]bool{}
for _, file in p.parsed_files {
for imp in file.vfile.imports {
all[imp.mod] = true
}
}
return all
}
// all_enums returns all enums from all parsed files
pub fn (p CodeParser) all_enums() []code.Enum {
mut all := []code.Enum{}
for _, file in p.parsed_files {
all << file.vfile.enums()
}
return all
}
// // all_interfaces returns all interfaces from all parsed files
// pub fn (p CodeParser) all_interfaces() []code.Interface {
// mut all := []code.Interface{}
// for _, file in p.parsed_files {
// all << file.vfile.interfaces()
// }
// return all
// }

207
lib/core/codeparser/json_export.v Normal file
View File

@@ -0,0 +1,207 @@
module codeparser
import json
import incubaid.herolib.core.code
// JSON export structures
pub struct CodeParserJSON {
pub mut:
root_dir string
modules map[string]ModuleJSON
summary SummaryJSON
}
pub struct ModuleJSON {
pub mut:
name string
files map[string]FileJSON
stats ModuleStats
imports []string
}
pub struct FileJSON {
pub:
path string
module_name string
items_count int
structs []StructJSON
functions []FunctionJSON
interfaces []InterfaceJSON
enums []EnumJSON
constants []ConstJSON
}
pub struct StructJSON {
pub:
name string
is_pub bool
field_count int
description string
}
pub struct FunctionJSON {
pub:
name string
is_pub bool
has_return bool
params int
receiver string
}
pub struct InterfaceJSON {
pub:
name string
is_pub bool
description string
}
pub struct EnumJSON {
pub:
name string
is_pub bool
value_count int
description string
}
pub struct ConstJSON {
pub:
name string
value string
}
pub struct SummaryJSON {
pub mut:
total_files int
total_modules int
total_structs int
total_functions int
total_interfaces int
total_enums int
}
// to_json exports the complete code structure to JSON
//
// Args:
// module_name - optional module filter (if empty, exports all modules)
// Returns:
// JSON string representation
pub fn (parser CodeParser) to_json(module_name string) !string {
mut result := CodeParserJSON{
root_dir: parser.root_dir
modules: map[string]ModuleJSON{}
summary: SummaryJSON{}
}
modules_to_process := if module_name != '' {
if module_name in parser.modules {
[module_name]
} else {
return error('module \'${module_name}\' not found')
}
} else {
parser.list_modules()
}
for mod_name in modules_to_process {
file_paths := parser.modules[mod_name]
mut module_json := ModuleJSON{
name: mod_name
files: map[string]FileJSON{}
imports: []string{}
}
for file_path in file_paths {
if parsed_file := parser.parsed_files[file_path] {
vfile := parsed_file.vfile
// Build structs JSON
mut structs_json := []StructJSON{}
for struct_ in vfile.structs() {
structs_json << StructJSON{
name: struct_.name
is_pub: struct_.is_pub
field_count: struct_.fields.len
description: struct_.description
}
}
// Build functions JSON
mut functions_json := []FunctionJSON{}
for func in vfile.functions() {
functions_json << FunctionJSON{
name: func.name
is_pub: func.is_pub
has_return: func.has_return
params: func.params.len
receiver: func.receiver.typ.symbol()
}
}
// Build interfaces JSON
mut interfaces_json := []InterfaceJSON{}
for item in vfile.items {
if item is code.Interface {
iface := item as code.Interface
interfaces_json << InterfaceJSON{
name: iface.name
is_pub: iface.is_pub
description: iface.description
}
}
}
// Build enums JSON
mut enums_json := []EnumJSON{}
for enum_ in vfile.enums() {
enums_json << EnumJSON{
name: enum_.name
is_pub: enum_.is_pub
value_count: enum_.values.len
description: enum_.description
}
}
// Build constants JSON
mut consts_json := []ConstJSON{}
for const_ in vfile.consts {
consts_json << ConstJSON{
name: const_.name
value: const_.value
}
}
file_json := FileJSON{
path: file_path
module_name: vfile.mod
items_count: vfile.items.len
structs: structs_json
functions: functions_json
interfaces: interfaces_json
enums: enums_json
constants: consts_json
}
module_json.files[file_path] = file_json
// Add imports to module level
for imp in vfile.imports {
if imp.mod !in module_json.imports {
module_json.imports << imp.mod
}
}
// Update summary
result.summary.total_structs += structs_json.len
result.summary.total_functions += functions_json.len
result.summary.total_interfaces += interfaces_json.len
result.summary.total_enums += enums_json.len
}
}
module_json.stats = parser.get_module_stats(mod_name)
result.modules[mod_name] = module_json
result.summary.total_modules++
}
return json.encode_pretty(result)
}

118
lib/core/codeparser/listers.v Normal file
View File

@@ -0,0 +1,118 @@
module codeparser
import incubaid.herolib.core.code
// list_modules returns a list of all parsed module names
pub fn (parser CodeParser) list_modules() []string {
return parser.modules.keys()
}
pub fn (parser CodeParser) list_files() []string {
return parser.parsed_files.keys()
}
// list_files_in_module returns all file paths in a specific module
pub fn (parser CodeParser) list_files_in_module(module_name string) []string {
return parser.modules[module_name] or { []string{} }
}
// list_structs returns all structs in the codebase (optionally filtered by module)
pub fn (parser CodeParser) list_structs(module_name string) []code.Struct {
mut structs := []code.Struct{}
for _, parsed_file in parser.parsed_files {
// Skip if module filter is provided and doesn't match
if module_name != '' && parsed_file.module_name != module_name {
continue
}
file_structs := parsed_file.vfile.structs()
structs << file_structs
}
return structs
}
// list_functions returns all functions in the codebase (optionally filtered by module)
pub fn (parser CodeParser) list_functions(module_name string) []code.Function {
mut functions := []code.Function{}
for _, parsed_file in parser.parsed_files {
if module_name != '' && parsed_file.module_name != module_name {
continue
}
file_functions := parsed_file.vfile.functions()
functions << file_functions
}
return functions
}
// list_interfaces returns all interfaces in the codebase (optionally filtered by module)
pub fn (parser CodeParser) list_interfaces(module_name string) []code.Interface {
mut interfaces := []code.Interface{}
for _, parsed_file in parser.parsed_files {
if module_name != '' && parsed_file.module_name != module_name {
continue
}
// Extract interfaces from items
for item in parsed_file.vfile.items {
if item is code.Interface {
interfaces << item
}
}
}
return interfaces
}
// list_methods_on_struct returns all methods (receiver functions) for a struct
pub fn (parser CodeParser) list_methods_on_struct(struct_name string, module_name string) []code.Function {
mut methods := []code.Function{}
functions := parser.list_functions(module_name)
for func in functions {
// Check if function has a receiver of the matching type
receiver_type := func.receiver.typ.symbol()
if receiver_type.contains(struct_name) {
methods << func
}
}
return methods
}
// list_imports returns all unique imports used in the codebase (optionally filtered by module)
pub fn (parser CodeParser) list_imports(module_name string) []code.Import {
mut imports := map[string]code.Import{}
for _, parsed_file in parser.parsed_files {
if module_name != '' && parsed_file.module_name != module_name {
continue
}
for imp in parsed_file.vfile.imports {
imports[imp.mod] = imp
}
}
return imports.values()
}
// list_constants returns all constants in the codebase (optionally filtered by module)
pub fn (parser CodeParser) list_constants(module_name string) []code.Const {
mut consts := []code.Const{}
for _, parsed_file in parser.parsed_files {
if module_name != '' && parsed_file.module_name != module_name {
continue
}
consts << parsed_file.vfile.consts
}
return consts
}

64
lib/core/codeparser/testdata/functions.v vendored Normal file
View File

@@ -0,0 +1,64 @@
module testdata
import time
import json
// create_user creates a new user in the system
// Arguments:
// email: user email address
// username: unique username
// Returns: the created User or error
pub fn create_user(email string, username string) !User {
if email == '' {
return error('email cannot be empty')
}
if username == '' {
return error('username cannot be empty')
}
return User{
id: 1
email: email
username: username
active: true
created: time.now().str()
updated: time.now().str()
}
}
// get_user retrieves a user by ID
pub fn get_user(user_id int) ?User {
if user_id <= 0 {
return none
}
return User{
id: user_id
email: 'user_${user_id}@example.com'
username: 'user_${user_id}'
active: true
created: '2024-01-01'
updated: '2024-01-01'
}
}
// delete_user deletes a user from the system
pub fn delete_user(user_id int) ! {
if user_id <= 0 {
return error('invalid user id')
}
}
// Internal helper for validation
fn validate_email(email string) bool {
return email.contains('@')
}
// Process multiple users
fn batch_create_users(emails []string) ![]User {
mut users := []User{}
for email in emails {
user_name := email.split('@')[0]
user := create_user(email, user_name)!
users << user
}
return users
}

40
lib/core/codeparser/testdata/methods.v vendored Normal file
View File

@@ -0,0 +1,40 @@
module testdata
import time
// activate sets the user as active
pub fn (mut u User) activate() {
u.active = true
u.updated = time.now().str()
}
// deactivate sets the user as inactive
pub fn (mut u User) deactivate() {
u.active = false
u.updated = time.now().str()
}
// is_active returns whether the user is active
pub fn (u User) is_active() bool {
return u.active
}
// get_display_name returns the display name for the user
pub fn (u &User) get_display_name() string {
if u.username != '' {
return u.username
}
return u.email
}
// set_profile updates the user profile
pub fn (mut u User) set_profile(mut profile Profile) ! {
if profile.user_id != u.id {
return error('profile does not belong to this user')
}
}
// get_profile_info returns profile information as string
pub fn (p &Profile) get_profile_info() string {
return 'Bio: ${p.bio}, Followers: ${p.followers}'
}

49
lib/core/codeparser/testdata/models.v vendored Normal file
View File

@@ -0,0 +1,49 @@
module testdata
import time
import os
const app_version = '1.0.0'
const max_users = 1000
const default_timeout = 30
// User represents an application user
// It stores all information related to a user
// including contact and status information
pub struct User {
pub:
id int
email string
username string
pub mut:
active bool
created string
updated string
}
// Profile represents user profile information
pub struct Profile {
pub:
user_id int
bio string
avatar string
mut:
followers int
following int
pub mut:
verified bool
}
// Settings represents user settings
struct Settings {
pub:
theme_dark bool
language string
mut:
notifications_enabled bool
}
struct InternalConfig {
debug bool
log_level int
}

36
lib/core/codeparser/testdata/services/cache.v vendored Normal file
View File

@@ -0,0 +1,36 @@
module services
import time
// Cache represents in-memory cache
pub struct Cache {
pub mut:
max_size int = 1000
mut:
items map[string]string
}
// new creates a new cache instance
pub fn Cache.new() &Cache {
return &Cache{
items: map[string]string{}
}
}
// set stores a value in cache with TTL
pub fn (mut c Cache) set(key string, value string, ttl int) {
c.items[key] = value
}
// get retrieves a value from cache
pub fn (c &Cache) get(key string) ?string {
if key in c.items {
return c.items[key]
}
return none
}
// clear removes all items from cache
pub fn (mut c Cache) clear() {
c.items.clear()
}

49
lib/core/codeparser/testdata/services/database.v vendored Normal file
View File

@@ -0,0 +1,49 @@
module services
import time
// Database handles all database operations
pub struct Database {
pub:
host string
port int
pub mut:
connected bool
pool_size int = 10
}
// new creates a new database connection
pub fn Database.new(host string, port int) !Database {
mut db := Database{
host: host
port: port
connected: false
}
return db
}
// connect establishes database connection
pub fn (mut db Database) connect() ! {
if db.host == '' {
return error('host cannot be empty')
}
db.connected = true
}
// disconnect closes database connection
pub fn (mut db Database) disconnect() ! {
db.connected = false
}
// query executes a database query
pub fn (db &Database) query(ssql string) ![]map[string]string {
if !db.connected {
return error('database not connected')
}
return []map[string]string{}
}
// execute_command executes a command and returns rows affected
pub fn (db &Database) execute_command(cmd string) !int {
return 0
}

44
lib/core/codeparser/testdata/utils/helpers.v vendored Normal file
View File

@@ -0,0 +1,44 @@
module utils
import crypto.md5
// Helper functions for common operations
// sanitize_input removes potentially dangerous characters
pub fn sanitize_input(input string) string {
return input.replace('<', '').replace('>', '')
}
// validate_password checks if password meets requirements
pub fn validate_password(password string) bool {
return password.len >= 8
}
// hash_password creates a hash of the password
pub fn hash_password(password string) string {
return md5.sum(password.bytes()).hex()
}
// generate_token creates a random token
// It uses current time to generate unique tokens
fn generate_token() string {
return 'token_12345'
}
// convert_to_json converts a user to JSON
pub fn (u User) to_json() string {
return '{}'
}
// compare_emails checks if two emails are the same
pub fn compare_emails(email1 string, email2 string) bool {
return email1.to_lower() == email2.to_lower()
}
// truncate_string limits string to max length
fn truncate_string(text string, max_len int) string {
if text.len > max_len {
return text[..max_len]
}
return text
}

26
lib/core/codeparser/testdata/utils/validators.v vendored Normal file
View File

@@ -0,0 +1,26 @@
module utils
// Email pattern validator
pub fn is_valid_email(email string) bool {
return email.contains('@') && email.contains('.')
}
// Phone number validator
pub fn is_valid_phone(phone string) bool {
return phone.len >= 10
}
// ID validator
fn is_valid_id(id int) bool {
return id > 0
}
// Check if string is alphanumeric
pub fn is_alphanumeric(text string) bool {
for c in text {
if !(c.is_alnum()) {
return false
}
}
return true
}

80
lib/core/flows/coordinator.v Normal file
View File

@@ -0,0 +1,80 @@
module flows
// __global (
// contexts map[u32]&Context
// context_current u32
// )
//
//
import incubaid.herolib.core.logger
import incubaid.herolib.ai.client as aiclient
import incubaid.herolib.core.redisclient
import incubaid.herolib.data.paramsparser
import incubaid.herolib.core.texttools
@[heap]
pub struct Coordinator {
pub mut:
name string
current_step string // links to steps dict
steps map[string]&Step
logger logger.Logger
ai ?aiclient.AIClient
redis ?&redisclient.Redis
}
@[params]
pub struct CoordinatorArgs {
pub mut:
name string @[required]
redis ?&redisclient.Redis
ai ?aiclient.AIClient = none
}
pub fn new(args CoordinatorArgs) !Coordinator {
ai := args.ai
return Coordinator{
name: args.name
logger: logger.new(path: '/tmp/flowlogger')!
ai: ai
redis: args.redis
}
}
@[params]
pub struct StepNewArgs {
pub mut:
name string
description string
f fn (mut s Step) ! @[required]
context map[string]string
error_steps []string
next_steps []string
error string
params paramsparser.Params
}
// add step to it
pub fn (mut c Coordinator) step_new(args StepNewArgs) !&Step {
mut s := Step{
coordinator: &c
name: args.name
description: args.description
main_step: args.f
error_steps: args.error_steps
next_steps: args.next_steps
error: args.error
params: args.params
}
s.name = texttools.name_fix(s.name)
c.steps[s.name] = &s
c.current_step = s.name
return &s
}
pub fn (mut c Coordinator) step_current() !&Step {
return c.steps[c.current_step] or {
return error('Current step "${c.current_step}" not found in coordinator "${c.name}"')
}
}

101
lib/core/flows/run.v Normal file
View File

@@ -0,0 +1,101 @@
module flows
import time as ostime
// Run the entire flow starting from current_step
pub fn (mut c Coordinator) run() ! {
mut s := c.step_current()!
c.run_step(mut s)!
}
// Run a single step, including error and next steps
pub fn (mut c Coordinator) run_step(mut step Step) ! {
// Initialize step
step.status = .running
step.started_at = ostime.now().unix_milli()
step.store_redis()!
// Log step start
step.log(
logtype: .stdout
log: 'Step "${step.name}" started'
)!
// Execute main step function
step.main_step(mut step) or {
// Handle error
step.status = .error
step.error_msg = err.msg()
step.finished_at = ostime.now().unix_milli()
step.store_redis()!
step.log(
logtype: .error
log: 'Step "${step.name}" failed: ${err.msg()}'
)!
// Run error steps if any
if step.error_steps.len > 0 {
for error_step_name in step.error_steps {
mut error_step := c.steps[error_step_name] or {
return error('Error step "${error_step_name}" not found in coordinator "${c.name}"')
}
c.run_step(mut error_step)!
}
}
return err
}
// Mark as success
step.status = .success
step.finished_at = ostime.now().unix_milli()
step.store_redis()!
step.log(
logtype: .stdout
log: 'Step "${step.name}" completed successfully'
)!
// Run next steps if any
if step.next_steps.len > 0 {
for next_step_name in step.next_steps {
mut next_step := c.steps[next_step_name] or {
return error('Next step "${next_step_name}" not found in coordinator "${c.name}"')
}
c.run_step(mut next_step)!
}
}
}
// Get step state from redis
pub fn (c Coordinator) get_step_state(step_name string) !map[string]string {
if mut redis := c.redis {
return redis.hgetall('flow:${c.name}:${step_name}')!
}
return error('Redis not configured')
}
// Get all steps state from redis (for UI dashboard)
pub fn (c Coordinator) get_all_steps_state() ![]map[string]string {
mut states := []map[string]string{}
if mut redis := c.redis {
pattern := 'flow:${c.name}:*'
keys := redis.keys(pattern)!
for key in keys {
state := redis.hgetall(key)!
states << state
}
}
return states
}
pub fn (c Coordinator) clear_redis() ! {
if mut redis := c.redis {
pattern := 'flow:${c.name}:*'
keys := redis.keys(pattern)!
for key in keys {
redis.del(key)!
}
}
}

91
lib/core/flows/step.v Normal file
View File

@@ -0,0 +1,91 @@
module flows
import incubaid.herolib.data.paramsparser
import incubaid.herolib.core.logger
import time as ostime
import json
pub enum StepStatus {
pending
running
success
error
skipped
}
pub struct Step {
pub mut:
status StepStatus = .pending
started_at i64 // Unix timestamp
finished_at i64
error_msg string
name string
description string
main_step fn (mut s Step) ! @[required]
context map[string]string
error_steps []string
next_steps []string
error string
logs []logger.LogItem
params paramsparser.Params
coordinator &Coordinator
}
pub fn (mut s Step) error_step_add(s2 &Step) {
s.error_steps << s2.name
}
pub fn (mut s Step) next_step_add(s2 &Step) {
s.next_steps << s2.name
}
pub fn (mut s Step) log(l logger.LogItemArgs) ! {
mut l2 := s.coordinator.logger.log(l)!
s.logs << l2
}
pub fn (mut s Step) store_redis() ! {
if mut redis := s.coordinator.redis {
key := 'flow:${s.coordinator.name}:${s.name}'
redis.hset(key, 'name', s.name)!
redis.hset(key, 'description', s.description)!
redis.hset(key, 'status', s.status.str())!
redis.hset(key, 'error', s.error_msg)!
redis.hset(key, 'logs_count', s.logs.len.str())!
redis.hset(key, 'started_at', s.started_at.str())!
redis.hset(key, 'finished_at', s.finished_at.str())!
redis.hset(key, 'json', s.to_json()!)!
// Set expiration to 24 hours
redis.expire(key, 86400)!
}
}
@[json: id]
pub struct StepJSON {
pub:
name string
description string
status string
error string
logs_count int
started_at i64
finished_at i64
duration i64 // milliseconds
}
pub fn (s Step) to_json() !string {
duration := s.finished_at - s.started_at
step_json := StepJSON{
name: s.name
description: s.description
status: s.status.str()
error: s.error_msg
logs_count: s.logs.len
started_at: s.started_at
finished_at: s.finished_at
duration: duration
}
return json.encode(step_json)
}

22
lib/core/generator/heromodels/ai_instructions.v Normal file
View File

@@ -0,0 +1,22 @@
module heromodels
import incubaid.herolib.develop.gittools
import incubaid.herolib.core.pathlib
pub fn aiprompts_path() !string {
return gittools.path(
git_url: 'https://github.com/Incubaid/herolib/tree/development/aiprompts'
)!.path
}
pub fn ai_instructions_hero_models() !string {
path := '${aiprompts_path()!}/ai_instructions_hero_models.md'
mut ppath := pathlib.get_file(path: path, create: false)!
return ppath.read()!
}
pub fn ai_instructions_vlang_herolib_core() !string {
path := '${aiprompts_path()!}/vlang_herolib_core.md'
mut ppath := pathlib.get_file(path: path, create: false)!
return ppath.read()!
}

182
lib/core/generator/heromodels/code_generator.v Executable file
View File

@@ -0,0 +1,182 @@
module heromodels
import incubaid.herolib.core.pathlib
import incubaid.herolib.ui.console
import incubaid.herolib.ai.client
import os
pub fn do() {
console.print_header('Code Generator - V File Analyzer Using AI')
// Find herolib root directory using @FILE
script_dir := os.dir(@FILE)
// Navigate from examples/core/code to root: up 4 levels
herolib_root := os.dir(os.dir(os.dir(script_dir)))
console.print_item('HeroLib Root: ${herolib_root}')
// The directory we want to analyze (lib/core in this case)
target_dir := herolib_root + '/lib/core'
console.print_item('Target Directory: ${target_dir}')
console.print_lf(1)
// Load instruction files from aiprompts
console.print_item('Loading instruction files...')
mut ai_instructions_file := pathlib.get(herolib_root +
'/aiprompts/ai_instructions_hero_models.md')
mut vlang_core_file := pathlib.get(herolib_root + '/aiprompts/vlang_herolib_core.md')
ai_instructions_content := ai_instructions_file.read()!
vlang_core_content := vlang_core_file.read()!
console.print_green('✓ Instruction files loaded successfully')
console.print_lf(1)
// Initialize AI client
console.print_item('Initializing AI client...')
mut aiclient := client.new()!
console.print_green('✓ AI client initialized')
console.print_lf(1)
// Get all V files from target directory
console.print_item('Scanning directory for V files...')
mut target_path := pathlib.get_dir(path: target_dir, create: false)!
mut all_files := target_path.list(
regex: [r'\.v$']
recursive: true
)!
console.print_item('Found ${all_files.paths.len} total V files')
// TODO: Walk over all files which do NOT end with _test.v and do NOT start with factory
// Each file becomes a src_file_content object
mut files_to_process := []pathlib.Path{}
for file in all_files.paths {
file_name := file.name()
// Skip test files
if file_name.ends_with('_test.v') {
continue
}
// Skip factory files
if file_name.starts_with('factory') {
continue
}
files_to_process << file
}
console.print_green('✓ After filtering: ${files_to_process.len} files to process')
console.print_lf(2)
// Process each file with AI
total_files := files_to_process.len
for idx, mut file in files_to_process {
current_idx := idx + 1
process_file_with_ai(mut aiclient, mut file, ai_instructions_content, vlang_core_content,
current_idx, total_files)!
}
console.print_lf(1)
console.print_header('✓ Code Generation Complete')
console.print_item('Processed ${files_to_process.len} files')
console.print_lf(1)
}
fn process_file_with_ai(mut aiclient client.AIClient, mut file pathlib.Path, ai_instructions string, vlang_core string, current int, total int) ! {
file_name := file.name()
src_file_path := file.absolute()
console.print_item('[${current}/${total}] Analyzing: ${file_name}')
// Read the file content - this is the src_file_content
src_file_content := file.read()!
// Build comprehensive system prompt
// TODO: Load instructions from prompt files and use in prompt
// Build the user prompt with context
user_prompt := '
File: ${file_name}
Path: ${src_file_path}
Current content:
\`\`\`v
${src_file_content}
\`\`\`
Please improve this V file by:
1. Following V language best practices
2. Ensuring proper error handling with ! and or blocks
3. Adding clear documentation comments
4. Following herolib patterns and conventions
5. Improving code clarity and readability
Context from herolib guidelines:
VLANG HEROLIB CORE:
${vlang_core}
AI INSTRUCTIONS FOR HERO MODELS:
${ai_instructions}
Return ONLY the complete improved file wrapped in \`\`\`v code block.
'
console.print_debug_title('Sending to AI', 'Calling AI model to improve ${file_name}...')
// TODO: Call AI client with model gemini-3-pro
aiclient.write_from_prompt(file, user_prompt, [.pro]) or {
console.print_stderr('Error processing ${file_name}: ${err}')
return
}
mut improved_file := pathlib.get(src_file_path + '.improved')
improved_content := improved_file.read()!
// Display improvements summary
sample_chars := 250
preview := if improved_content.len > sample_chars {
improved_content[..sample_chars] + '... (preview truncated)'
} else {
improved_content
}
console.print_debug_title('AI Analysis Results for ${file_name}', preview)
// Optional: Save improved version for review
// Uncomment to enable saving
// improved_file_path := src_file_path + '.improved'
// mut improved_file := pathlib.get_file(path: improved_file_path, create: true)!
// improved_file.write(improved_content)!
// console.print_green('✓ Improvements saved to: ${improved_file_path}')
console.print_lf(1)
}
// Extract V code from markdown code block
fn extract_code_block(response string) string {
// Look for ```v ... ``` block
start_marker := '\`\`\`v'
end_marker := '\`\`\`'
start_idx := response.index(start_marker) or {
// If no ```v, try to return as-is
return response
}
mut content_start := start_idx + start_marker.len
if content_start < response.len && response[content_start] == `\n` {
content_start++
}
end_idx := response.index(end_marker) or { return response[content_start..] }
extracted := response[content_start..end_idx]
return extracted.trim_space()
}

25
lib/core/generator/heromodels/templates/model_code.md Normal file
View File

@@ -0,0 +1,25 @@
File: ${file_name}
Path: ${src_file_path}
Current content:
```v
${src_file_content}
```
Please improve this V file by:
1. Following V language best practices
2. Ensuring proper error handling with ! and or blocks
3. Adding clear documentation comments
4. Following herolib patterns and conventions
5. Improving code clarity and readability
Context from herolib guidelines:
VLANG HEROLIB CORE:
${vlang_core}
AI INSTRUCTIONS FOR HERO MODELS:
${ai_instructions}
Return ONLY the complete improved file wrapped in ```v code block.

9
lib/core/logger/log.v
View File

@@ -14,7 +14,7 @@ pub mut:
logtype LogType
}
pub fn (mut l Logger) log(args_ LogItemArgs) ! {
pub fn (mut l Logger) log(args_ LogItemArgs) !LogItem {
mut args := args_
t := args.timestamp or {
@@ -67,6 +67,13 @@ pub fn (mut l Logger) log(args_ LogItemArgs) ! {
if l.console_output {
l.write_to_console(args, t)!
}
return LogItem{
timestamp: t
cat: args.cat
log: args.log
logtype: args.logtype
}
}
// Write log message to console with clean formatting

142
lib/core/pathlib/path_list.v
View File

@@ -1,60 +1,69 @@
module pathlib
import os
import regex
// import incubaid.herolib.core.smartid
import incubaid.herolib.ui.console
import incubaid.herolib.core.texttools.regext
@[params]
pub struct ListArgs {
pub mut:
regex []string
recursive bool = true
ignore_default bool = true // ignore files starting with . and _
include_links bool // wether to include links in list
dirs_only bool
files_only bool
// Include if matches any regex pattern
regex []string
// Exclude if matches any regex pattern
regex_ignore []string
// Include if matches any wildcard pattern (* = any sequence)
filter []string
// Exclude if matches any wildcard pattern
filter_ignore []string
// Traverse directories recursively
recursive bool = true
// Ignore files starting with . and _
ignore_default bool = true
// Include symlinks
include_links bool
// Return only directories
dirs_only bool
// Return only files
files_only bool
}
// the result of pathlist
// Result of list operation
pub struct PathList {
pub mut:
// is the root under which all paths are, think about it like a changeroot environment
root string
// Root directory where listing started
root string
// Found paths
paths []Path
}
// list all files & dirs, follow symlinks .
// will sort all items .
// return as list of Paths .
// .
// params: .
// ```
// regex []string
// recursive bool = true // default true, means we recursive over dirs by default
// ignore_default bool = true // ignore files starting with . and _
// dirs_only bool
// List files and directories with filtering
//
// example see https://github.com/incubaid/herolib/blob/development/examples/core/pathlib/examples/list/path_list.v
// Parameters:
// - regex: Include if matches regex pattern (e.g., `r'.*\.v$'`)
// - regex_ignore: Exclude if matches regex pattern
// - filter: Include if matches wildcard pattern (e.g., `'*.txt'`, `'test*'`, `'config'`)
// - filter_ignore: Exclude if matches wildcard pattern
// - recursive: Traverse directories (default: true)
// - ignore_default: Ignore files starting with . and _ (default: true)
// - dirs_only: Return only directories
// - files_only: Return only files
// - include_links: Include symlinks in results
//
// e.g. p.list(regex:[r'.*\.v$'])! //notice the r in front of string, this is regex for all files ending with .v
//
// ```
// please note links are ignored for walking over dirstructure (for files and dirs)
// Examples:
// dir.list(regex: [r'.*\.v$'], recursive: true)!
// dir.list(filter: ['*.txt', 'config*'], filter_ignore: ['*.bak'])!
// dir.list(regex: [r'.*test.*'], regex_ignore: [r'.*_test\.v$'])!
pub fn (mut path Path) list(args_ ListArgs) !PathList {
// $if debug {
// console.print_header(' list: ${args_}')
// }
mut r := []regex.RE{}
for regexstr in args_.regex {
mut re := regex.regex_opt(regexstr) or {
return error("cannot create regex for:'${regexstr}'")
}
// console.print_debug(re.get_query())
r << re
}
// Create matcher from the list arguments - handles all regex and wildcard conversions
matcher := regext.new(
regex: args_.regex
regex_ignore: args_.regex_ignore
filter: args_.filter
filter_ignore: args_.filter_ignore
)!
mut args := ListArgsInternal{
regex: r
matcher: matcher
recursive: args_.recursive
ignore_default: args_.ignore_default
dirs_only: args_.dirs_only
@@ -70,11 +79,11 @@ pub fn (mut path Path) list(args_ ListArgs) !PathList {
}
@[params]
pub struct ListArgsInternal {
struct ListArgsInternal {
mut:
regex []regex.RE // only put files in which follow one of the regexes
matcher regext.Matcher
recursive bool = true
ignore_default bool = true // ignore files starting with . and _
ignore_default bool = true
dirs_only bool
files_only bool
include_links bool
@@ -85,7 +94,6 @@ fn (mut path Path) list_internal(args ListArgsInternal) ![]Path {
path.check()
if !path.is_dir() && (!path.is_dir_link() || !args.include_links) {
// return error('Path must be directory or link to directory')
return []Path{}
}
if debug {
@@ -94,27 +102,33 @@ fn (mut path Path) list_internal(args ListArgsInternal) ![]Path {
mut ls_result := os.ls(path.path) or { []string{} }
ls_result.sort()
mut all_list := []Path{}
for item in ls_result {
if debug {
console.print_stdout(' - ${item}')
}
p := os.join_path(path.path, item)
mut new_path := get(p)
// Check for dir and linkdir
// Check for broken symlinks
if !new_path.exists() {
// to deal with broken link
continue
}
// Skip symlinks if not included
if new_path.is_link() && !args.include_links {
continue
}
// Skip hidden/underscore files if ignore_default
if args.ignore_default {
if item.starts_with('_') || item.starts_with('.') {
continue
}
}
// Process directories
if new_path.is_dir() || (new_path.is_dir_link() && args.include_links) {
// If recusrive
if args.recursive {
mut rec_list := new_path.list_internal(args)!
all_list << rec_list
@@ -126,20 +140,8 @@ fn (mut path Path) list_internal(args ListArgsInternal) ![]Path {
}
}
mut addthefile := false
// If no regex patterns provided, include all files
if args.regex.len == 0 {
addthefile = true
} else {
// Include file if ANY regex pattern matches (OR operation)
for r in args.regex {
if r.matches_string(item) {
addthefile = true
break
}
}
}
if addthefile && !args.dirs_only {
// Use matcher to check if file matches include/exclude patterns
if args.matcher.match(item) && !args.dirs_only {
if !args.files_only || new_path.is_file() {
all_list << new_path
}
@@ -148,34 +150,16 @@ fn (mut path Path) list_internal(args ListArgsInternal) ![]Path {
return all_list
}
// copy all
// Copy all paths to destination directory
pub fn (mut pathlist PathList) copy(dest string) ! {
for mut path in pathlist.paths {
path.copy(dest: dest)!
}
}
// delete all
// Delete all paths
pub fn (mut pathlist PathList) delete() ! {
for mut path in pathlist.paths {
path.delete()!
}
}
// sids_acknowledge .
// pub fn (mut pathlist PathList) sids_acknowledge(cid smartid.CID) ! {
// for mut path in pathlist.paths {
// path.sids_acknowledge(cid)!
// }
// }
// // sids_replace .
// // find parts of text in form sid:*** till sid:****** .
// // replace all occurrences with new sid's which are unique .
// // cid = is the circle id for which we find the id's .
// // sids will be replaced in the files if they are different
// pub fn (mut pathlist PathList) sids_replace(cid smartid.CID) ! {
// for mut path in pathlist.paths {
// path.sids_replace(cid)!
// }
// }

65
lib/core/pathlib/path_tools.v
View File

@@ -2,6 +2,7 @@ module pathlib
import os
import incubaid.herolib.core.texttools
import incubaid.herolib.core.texttools.regext
import time
import crypto.md5
import rand
@@ -292,6 +293,70 @@ pub fn (path Path) parent_find(tofind string) !Path {
return path2.parent_find(tofind)
}
// parent_find_advanced walks up the directory tree, collecting all items that match tofind
// pattern until it encounters an item matching the stop pattern.
// Both tofind and stop use matcher filter format supporting wildcards:
// - '*.txt' matches any .txt file
// - 'src*' matches anything starting with 'src'
// - '.git' matches exactly '.git'
// - '*test*' matches anything containing 'test'
//
// Returns all found paths before hitting the stop condition.
// If stop is never found, continues until reaching filesystem root.
//
// Examples:
// // Find all 'test_*.v' files until reaching '.git' directory
// tests := my_path.parent_find_advanced('test_*.v', '.git')!
//
// // Find any 'Makefile*' until hitting 'node_modules'
// makefiles := my_path.parent_find_advanced('Makefile*', 'node_modules')!
//
// // Find '*.md' files until reaching '.git'
// docs := my_path.parent_find_advanced('*.md', '.git')!
pub fn (path Path) parent_find_advanced(tofind string, stop string) ![]Path {
// Start from current path or its parent if it's a file
mut search_path := path
if search_path.is_file() {
search_path = search_path.parent()!
}
// Create matchers from filter patterns
tofind_matcher := regext.new(filter: [tofind])!
stop_matcher := regext.new(filter: [stop])!
mut found_paths := []Path{}
mut current := search_path
for {
// List contents of current directory
mut items := os.ls(current.path) or { []string{} }
// Check each item in the directory
for item in items {
// Check if this is the stop pattern - if yes, halt and return
if stop_matcher.match(item) {
return found_paths
}
// Check if this matches what we're looking for
if tofind_matcher.match(item) {
full_path := os.join_path(current.path, item)
mut found_path := get(full_path)
if found_path.exists() {
found_paths << found_path
}
}
}
// Try to move to parent directory
current = current.parent() or {
// Reached filesystem root, return what we found
return found_paths
}
}
return found_paths
}
// delete
pub fn (mut path Path) rm() ! {
return path.delete()

277
lib/core/pathlib/readme.md
View File

@@ -1,7 +1,5 @@
# Pathlib Module
The pathlib module provides a robust way to handle file system operations. Here's a comprehensive overview of how to use it:
## 1. Basic Path Creation
```v
@@ -45,50 +43,121 @@ if path.is_link() { /* is symlink */ }
## 3. File Listing and Filtering
```v
// List all files in a directory (recursive by default)
mut dir := pathlib.get('/some/dir')
mut pathlist := dir.list()!
### 3.1 Regex-Based Filtering
// List only files matching specific extensions using regex
mut pathlist_images := dir.list(
regex: [r'.*\.png$', r'.*\.jpg$', r'.*\.svg$', r'.*\.jpeg$'],
```v
import incubaid.herolib.core.pathlib
mut dir := pathlib.get('/some/code/project')
// Include files matching regex pattern (e.g., all V files)
mut v_files := dir.list(
regex: [r'.*\.v$']
)!
// Multiple regex patterns (OR logic)
mut source_files := dir.list(
regex: [r'.*\.v$', r'.*\.ts$', r'.*\.go$']
)!
// Exclude certain patterns
mut no_tests := dir.list(
regex: [r'.*\.v$'],
regex_ignore: [r'.*_test\.v$']
)!
// Ignore both default patterns and custom ones
mut important_files := dir.list(
regex: [r'.*\.v$'],
regex_ignore: [r'.*_test\.v$', r'.*\.bak$']
)!
```
### 3.2 Simple String-Based Filtering
```v
import incubaid.herolib.core.pathlib
mut dir := pathlib.get('/some/project')
// Include files/dirs containing string in name
mut config_files := dir.list(
contains: ['config']
)!
// Multiple contains patterns (OR logic)
mut important := dir.list(
contains: ['main', 'core', 'config'],
recursive: true
)!
// Exclude files containing certain strings
mut no_backups := dir.list(
contains_ignore: ['.bak', '.tmp', '.backup']
)!
// Combine contains with exclude
mut python_but_no_cache := dir.list(
contains: ['.py'],
contains_ignore: ['__pycache__', '.pyc']
)!
```
### 3.3 Advanced Filtering Options
```v
import incubaid.herolib.core.pathlib
mut dir := pathlib.get('/some/project')
// List only directories
mut pathlist_dirs := dir.list(
mut dirs := dir.list(
dirs_only: true,
recursive: true
)!
// List only files
mut pathlist_files := dir.list(
mut files := dir.list(
files_only: true,
recursive: false // only in current directory
recursive: false
)!
// Include symlinks in the results
mut pathlist_with_links := dir.list(
// Include symlinks
mut with_links := dir.list(
regex: [r'.*\.conf$'],
include_links: true
)!
// Don't ignore hidden files (those starting with . or _)
mut pathlist_all := dir.list(
ignore_default: false
// Don't ignore hidden files (starting with . or _)
mut all_files := dir.list(
ignore_default: false,
recursive: true
)!
// Non-recursive (only in current directory)
mut immediate := dir.list(
recursive: false
)!
// Access the resulting paths
for path in pathlist.paths {
println(path.path)
for path in dirs.paths {
println('${path.name()}')
}
// Perform operations on all paths in the list
pathlist.copy('/destination/dir')!
pathlist.delete()!
```
## 4. Common File Operations
## 4. Path Operations on Lists
```v
mut pathlist := dir.list(regex: [r'.*\.tmp$'])!
// Delete all files matching filter
pathlist.delete()!
// Copy all files to destination
pathlist.copy('/backup/location')!
```
## 5. Common File Operations
```v
// Empty a directory
@@ -107,67 +176,117 @@ mut path := pathlib.get_dir(
mut wd := pathlib.get_wd()
```
## Features
## 6. Path Scanning with Filters and Executors
The module handles common edge cases:
Path scanning processes directory trees with custom filter and executor functions.
- Automatically expands ~ to home directory
- Creates parent directories as needed
- Provides proper error handling with V's result type
- Checks path existence and type
- Handles both absolute and relative paths
### 6.1 Basic Scanner Usage
## Path Object Structure
```v
import incubaid.herolib.core.pathlib
import incubaid.herolib.data.paramsparser
// Define a filter function (return true to continue processing)
fn my_filter(mut path pathlib.Path, mut params paramsparser.Params) !bool {
// Skip files larger than 1MB
size := path.size()!
return size < 1_000_000
}
// Define an executor function (process the file)
fn my_executor(mut path pathlib.Path, mut params paramsparser.Params) !paramsparser.Params {
if path.is_file() {
content := path.read()!
println('Processing: ${path.name()} (${content.len} bytes)')
}
return params
}
// Run the scan
mut root := pathlib.get_dir(path: '/source/dir')!
mut params := paramsparser.new_params()
root.scan(mut params, [my_filter], [my_executor])!
```
### 6.2 Scanner with Multiple Filters and Executors
```v
import incubaid.herolib.core.pathlib
import incubaid.herolib.data.paramsparser
// Filter 1: Skip hidden files
fn skip_hidden(mut path pathlib.Path, mut params paramsparser.Params) !bool {
return !path.name().starts_with('.')
}
// Filter 2: Only process V files
fn only_v_files(mut path pathlib.Path, mut params paramsparser.Params) !bool {
if path.is_file() {
return path.extension() == 'v'
}
return true
}
// Executor 1: Count lines
fn count_lines(mut path pathlib.Path, mut params paramsparser.Params) !paramsparser.Params {
if path.is_file() {
content := path.read()!
lines := content.split_into_lines().len
params.set('total_lines', (params.get_default('total_lines', '0').int() + lines).str())
}
return params
}
// Executor 2: Print file info
fn print_info(mut path pathlib.Path, mut params paramsparser.Params) !paramsparser.Params {
if path.is_file() {
size := path.size()!
println('${path.name()}: ${int(size)} bytes')
}
return params
}
// Run scan with all filters and executors
mut root := pathlib.get_dir(path: '/source/code')!
mut params := paramsparser.new_params()
root.scan(mut params, [skip_hidden, only_v_files], [count_lines, print_info])!
total := params.get('total_lines')!
println('Total lines: ${total}')
```
## 7. Sub-path Getters and Checkers
```v
// Get a sub-path with name fixing and case-insensitive matching
path.sub_get(name: 'mysub_file.md', name_fix_find: true, name_fix: true)!
// Check if a sub-path exists
path.sub_exists(name: 'my_sub_dir')!
// File operations
path.file_exists('file.txt') // bool
path.file_exists_ignorecase('File.Txt') // bool
path.file_get('file.txt')! // Path
path.file_get_ignorecase('File.Txt')! // Path
path.file_get_new('new.txt')! // Get or create
// Directory operations
path.dir_exists('mydir') // bool
path.dir_get('mydir')! // Path
path.dir_get_new('newdir')! // Get or create
// Symlink operations
path.link_exists('mylink') // bool
path.link_get('mylink')! // Path
```
## 8. Path Object Structure
Each Path object contains:
- `path`: The actual path string
- `cat`: Category (file/dir/link)
- `exist`: Existence status
- `cat`: Category (file/dir/linkfile/linkdir)
- `exist`: Existence status (yes/no/unknown)
This provides a safe and convenient API for all file system operations in V.
## 5. Sub-path Getters and Checkers
The `pathlib` module provides methods to get and check for the existence of sub-paths (files, directories, and links) within a given path.
```v
// Get a sub-path (file or directory) with various options
path.sub_get(name:"mysub_file.md", name_fix_find:true, name_fix:true)!
// Check if a sub-path exists
path.sub_exists(name:"my_sub_dir")!
// Check if a file exists
path.file_exists("my_file.txt")
// Check if a file exists (case-insensitive)
path.file_exists_ignorecase("My_File.txt")
// Get a file as a Path object
path.file_get("another_file.txt")!
// Get a file as a Path object (case-insensitive)
path.file_get_ignorecase("Another_File.txt")!
// Get a file, create if it doesn't exist
path.file_get_new("new_file.txt")!
// Check if a link exists
path.link_exists("my_link")
// Check if a link exists (case-insensitive)
path.link_exists_ignorecase("My_Link")
// Get a link as a Path object
path.link_get("some_link")!
// Check if a directory exists
path.dir_exists("my_directory")
// Get a directory as a Path object
path.dir_get("another_directory")!
// Get a directory, create if it doesn't exist
path.dir_get_new("new_directory")!
```

203
lib/core/texttools/regext/matcher.v Normal file
View File

@@ -0,0 +1,203 @@
module regext
import regex
// Arguments for creating a matcher
@[params]
pub struct MatcherArgs {
pub mut:
// Include if matches any regex pattern
regex []string
// Exclude if matches any regex pattern
regex_ignore []string
// Include if matches any wildcard pattern (* = any sequence)
filter []string
// Exclude if matches any wildcard pattern
filter_ignore []string
}
// Matcher matches strings against include/exclude regex patterns
pub struct Matcher {
mut:
regex_include []regex.RE
filter_include []regex.RE
regex_exclude []regex.RE
}
// Create a new matcher from arguments
//
// Parameters:
// - regex: Include if matches regex pattern (e.g., $r'.*\.v'$')
// - regex_ignore: Exclude if matches regex pattern
// - filter: Include if matches wildcard pattern (e.g., $r'*.txt'$, $r'test*'$, $r'config'$)
// - filter_ignore: Exclude if matches wildcard pattern
//
// Logic:
// - If both regex and filter patterns are provided, BOTH must match (AND logic)
// - If only regex patterns are provided, any regex pattern can match (OR logic)
// - If only filter patterns are provided, any filter pattern can match (OR logic)
// - Exclude patterns take precedence over include patterns
//
// Examples:
// $m := regex.new(regex: [r'.*\.v$'])!$
// $m := regex.new(filter: ['*.txt'], filter_ignore: ['*.bak'])!$
// $m := regex.new(regex: [r'.*test.*'], regex_ignore: [r'.*_test\.v$'])!$
pub fn new(args_ MatcherArgs) !Matcher {
mut regex_include := []regex.RE{}
mut filter_include := []regex.RE{}
// Add regex patterns
for regexstr in args_.regex {
mut re := regex.regex_opt(regexstr) or {
return error("cannot create regex for:'${regexstr}'")
}
regex_include << re
}
// Convert wildcard filters to regex and add separately
for filter_pattern in args_.filter {
mut has_wildcards_in_original_filter := false
for r in filter_pattern.runes() {
if r == `*` || r == `?` {
has_wildcards_in_original_filter = true
break
}
}
regex_pattern := wildcard_to_regex(filter_pattern)
mut re := regex.regex_opt(regex_pattern) or {
return error("cannot create regex from filter:'${filter_pattern}'")
}
// Explicitly set f_ms and f_me flags for exact matches if no wildcards were in the original pattern
if !has_wildcards_in_original_filter {
re.flag |= regex.f_ms // Match string start
re.flag |= regex.f_me // Match string end
}
filter_include << re
}
mut regex_exclude := []regex.RE{}
// Add regex ignore patterns
for regexstr in args_.regex_ignore {
mut re := regex.regex_opt(regexstr) or {
return error("cannot create ignore regex for:'${regexstr}'")
}
regex_exclude << re
}
// Convert wildcard ignore filters to regex and add
for filter_pattern in args_.filter_ignore {
// For ignore patterns, no special f_ms/f_me flags are needed, default wildcard_to_regex behavior is sufficient
regex_pattern := wildcard_to_regex(filter_pattern)
mut re := regex.regex_opt(regex_pattern) or {
return error("cannot create ignore regex from filter:'${filter_pattern}'")
}
regex_exclude << re
}
return Matcher{
regex_include: regex_include
filter_include: filter_include
regex_exclude: regex_exclude
}
}
// match checks if a string matches the include patterns and not the exclude patterns
//
// Logic:
// - If both regex and filter patterns exist, string must match BOTH (AND logic)
// - If only regex patterns exist, string must match at least one (OR logic)
// - If only filter patterns exist, string must match at least one (OR logic)
// - Then check if string matches any exclude pattern; if yes, return false
// - Otherwise return true
//
// Examples:
// $m := regex.new(regex: [r'.*\.v$'])!$
// $result := m.match('file.v') // true$
// $result := m.match('file.txt') // false$
//
// $m2 := regex.new(filter: ['*.txt'], filter_ignore: ['*.bak'])!$
// $result := m2.match('readme.txt') // true$
// $result := m2.match('backup.bak') // false$
//
// $m3 := regex.new(filter: ['src*'], regex: [r'.*\.v$'])!$
// $result := m3.match('src/main.v') // true (matches both)$
// $result := m3.match('src/config.txt') // false (doesn't match regex)$
// $result := m3.match('main.v') // false (doesn't match filter)$
pub fn (m Matcher) match(text string) bool {
// Determine if we have both regex and filter patterns
has_regex := m.regex_include.len > 0
has_filter := m.filter_include.len > 0
// If both regex and filter patterns exist, string must match BOTH
if has_regex && has_filter {
mut regex_matched := false
for re in m.regex_include {
if re.matches_string(text) {
regex_matched = true
break
}
}
if !regex_matched {
return false
}
mut filter_matched := false
for re in m.filter_include {
if re.matches_string(text) {
filter_matched = true
break
}
}
if !filter_matched {
return false
}
} else if has_regex {
// Only regex patterns: string must match at least one
mut matched := false
for re in m.regex_include {
if re.matches_string(text) {
matched = true
break
}
}
if !matched {
return false
}
} else if has_filter {
// Only filter patterns: string must match at least one
mut matched := false
for re in m.filter_include {
if re.matches_string(text) {
matched = true
break
}
}
if !matched {
return false
}
} else {
// If no include patterns are defined, everything matches initially
// unless there are explicit exclude patterns.
// This handles the case where new() is called without any include patterns.
if m.regex_exclude.len == 0 {
return true // No includes and no excludes, so everything matches.
}
// If no include patterns but there are exclude patterns,
// we defer to the exclude patterns check below.
}
// Check exclude patterns - if matches any, return false
for re in m.regex_exclude {
if re.matches_string(text) {
return false
}
}
// If we reach here, it either matched includes (or no includes were set and
// no excludes were set, or no includes were set but it didn't match any excludes)
// and didn't match any excludes
return true
}

234
lib/core/texttools/regext/matcher_test.v Normal file
View File

@@ -0,0 +1,234 @@
module regext
fn test_matcher_no_constraints() {
m := new()!
assert m.match('file.txt') == true
assert m.match('anything.v') == true
assert m.match('') == true
assert m.match('test-123_file.log') == true
}
fn test_matcher_regex_include_single() {
m := new(regex: [r'.*\.v$'])!
assert m.match('file.v') == true
assert m.match('test.v') == true
assert m.match('main.v') == true
assert m.match('file.txt') == false
assert m.match('image.png') == false
assert m.match('file.v.bak') == false
}
fn test_matcher_regex_include_multiple() {
m := new(regex: [r'.*\.v$', r'.*\.txt$'])!
assert m.match('file.v') == true
assert m.match('readme.txt') == true
assert m.match('main.v') == true
assert m.match('notes.txt') == true
assert m.match('image.png') == false
assert m.match('archive.tar.gz') == false
}
fn test_matcher_regex_ignore_single() {
m := new(regex_ignore: [r'.*_test\.v$'])!
assert m.match('main.v') == true
assert m.match('helper.v') == true
assert m.match('file_test.v') == false
assert m.match('test_file.v') == true // doesn't end with _test.v
assert m.match('test_helper.txt') == true
}
fn test_matcher_regex_ignore_multiple() {
m := new(regex_ignore: [r'.*_test\.v$', r'.*\.bak$'])!
assert m.match('main.v') == true
assert m.match('file_test.v') == false
assert m.match('backup.bak') == false
assert m.match('old_backup.bak') == false
assert m.match('readme.txt') == true
assert m.match('test_data.bak') == false
}
fn test_matcher_regex_include_and_exclude() {
m := new(regex: [r'.*\.v$'], regex_ignore: [r'.*_test\.v$'])!
assert m.match('main.v') == true
assert m.match('helper.v') == true
assert m.match('file_test.v') == false
assert m.match('image.png') == false
assert m.match('test_helper.v') == true
assert m.match('utils_test.v') == false
}
fn test_matcher_filter_wildcard_start() {
m := new(filter: ['*.txt'])!
assert m.match('readme.txt') == true
assert m.match('config.txt') == true
assert m.match('notes.txt') == true
assert m.match('file.v') == false
assert m.match('.txt') == true
assert m.match('txt') == false
}
fn test_matcher_filter_wildcard_end() {
m := new(filter: ['test*'])!
assert m.match('test_file.v') == true
assert m.match('test') == true
assert m.match('test.txt') == true
assert m.match('file_test.v') == false
assert m.match('testing.v') == true
}
fn test_matcher_filter_substring() {
// FIXED: Updated assertions to reflect exact matching for filter patterns without explicit wildcards
m := new(filter: ['config'])!
assert m.match('config.txt') == false // Should not match, exact match is 'config'
assert m.match('my_config_file.v') == false // Should not match, exact match is 'config'
assert m.match('config') == true
assert m.match('reconfigure.py') == false // Should not match, exact match is 'config'
assert m.match('settings.txt') == false
}
fn test_matcher_filter_multiple() {
m := new(filter: ['*.v', '*.txt', 'config*'])!
assert m.match('main.v') == true
assert m.match('readme.txt') == true
assert m.match('config.yaml') == true
assert m.match('configuration.json') == true
assert m.match('image.png') == false
}
fn test_matcher_filter_with_exclude() {
// FIXED: Changed test to use *test* pattern instead of *_test.v
// This correctly excludes files containing 'test'
m := new(filter: ['*.v'], filter_ignore: ['*test*.v'])!
assert m.match('main.v') == true
assert m.match('helper.v') == true
assert m.match('helper_test.v') == false
assert m.match('file.txt') == false
assert m.match('test_helper.v') == false // Now correctly excluded
}
fn test_matcher_filter_ignore_multiple() {
m := new(filter: ['*'], filter_ignore: ['*.bak', '*_old.*'])!
assert m.match('file.txt') == true
assert m.match('main.v') == true
assert m.match('backup.bak') == false
assert m.match('config_old.v') == false
assert m.match('data_old.txt') == false
assert m.match('readme.md') == true
}
fn test_matcher_complex_combined() {
// FIXED: Refactored regex patterns to avoid token-level OR issues
m := new(
regex: [r'.*\.v$', r'.*\.go$', r'.*\.rs$']
regex_ignore: [r'.*test.*']
filter: ['src*']
filter_ignore: ['*_generated.*']
)!
assert m.match('src/main.v') == true
assert m.match('src/helper.go') == true
assert m.match('src/lib.rs') == true
assert m.match('src/main_test.v') == false
assert m.match('src/main_generated.rs') == false
assert m.match('main.v') == false
assert m.match('test/helper.v') == false
}
fn test_matcher_empty_patterns() {
m := new(regex: [r'.*\.v$'])!
assert m.match('') == false
m2 := new()!
assert m2.match('') == true
}
fn test_matcher_special_characters_in_wildcard() {
m := new(filter: ['*.test[1].v'])!
assert m.match('file.test[1].v') == true
assert m.match('main.test[1].v') == true
assert m.match('file.test1.v') == false
}
fn test_matcher_case_sensitive() {
// FIXED: Use proper regex anchoring to match full patterns
m := new(regex: [r'.*Main.*'])! // Match 'Main' anywhere in the string
assert m.match('Main.v') == true
assert m.match('main.v') == false
assert m.match('MAIN.v') == false
assert m.match('main_Main.txt') == true // Now correctly matches
}
fn test_matcher_exclude_takes_precedence() {
// If something matches include but also exclude, exclude wins
m := new(regex: [r'.*\.v$'], regex_ignore: [r'.*\.v$'])!
assert m.match('file.v') == false
assert m.match('file.txt') == false
}
fn test_matcher_only_exclude_allows_everything_except() {
m := new(regex_ignore: [r'.*\.bak$'])!
assert m.match('main.v') == true
assert m.match('file.txt') == true
assert m.match('config.py') == true
assert m.match('backup.bak') == false
assert m.match('old.bak') == false
}
fn test_matcher_complex_regex_patterns() {
// FIXED: Refactored regex patterns to avoid token-level OR issues
m := new(regex: [r'.*\.go$', r'.*\.v$', r'.*\.rs$', r'.*Makefile.*'])!
assert m.match('main.go') == true
assert m.match('main.v') == true
assert m.match('lib.rs') == true
assert m.match('Makefile') == true
assert m.match('Makefile.bak') == true
assert m.match('main.py') == false
}
fn test_matcher_wildcard_combinations() {
m := new(filter: ['src/*test*.v', '*_helper.*'])!
assert m.match('src/main_test.v') == true
assert m.match('src/test_utils.v') == true
assert m.match('utils_helper.js') == true
assert m.match('src/main.v') == false
assert m.match('test_helper.go') == true
}
fn test_matcher_edge_case_dot_files() {
// FIXED: Use correct regex escape sequence for dot files
m := new(regex_ignore: [r'^\..*'])! // Match files starting with dot
assert m.match('.env') == false
assert m.match('.gitignore') == false
assert m.match('file.dotfile') == true
assert m.match('main.v') == true
}
fn test_matcher_multiple_extensions() {
m := new(filter: ['*.tar.gz', '*.tar.bz2'])!
assert m.match('archive.tar.gz') == true
assert m.match('backup.tar.bz2') == true
assert m.match('file.gz') == false
assert m.match('file.tar') == false
}
fn test_matcher_path_like_strings() {
m := new(regex: [r'.*src/.*\.v$'])!
assert m.match('src/main.v') == true
assert m.match('src/utils/helper.v') == true
assert m.match('test/main.v') == false
assert m.match('src/config.txt') == false
}
fn test_matcher_filter_ignore_with_regex() {
// FIXED: When both filter and regex are used, they should both match (AND logic)
// This requires separating filter and regex include patterns
m := new(
filter: ['src*']
regex: [r'.*\.v$']
regex_ignore: [r'.*_temp.*']
)!
assert m.match('src/main.v') == true
assert m.match('src/helper.v') == true
assert m.match('src/main_temp.v') == false
assert m.match('src/config.txt') == false // Doesn't match .*\.v$ regex
assert m.match('main.v') == false // Doesn't match src* filter
}

109
lib/core/texttools/regext/readme.md
View File

@@ -1,15 +1,110 @@
# regex
## basic regex utilities
## escape_regex_chars
- .
Escapes special regex metacharacters in a string to make it safe for use in regex patterns.
```v
import incubaid.herolib.core.texttools.regext
escaped := regext.escape_regex_chars("file.txt")
// Result: "file\.txt"
// Use in regex patterns:
safe_search := regext.escape_regex_chars("[test]")
// Result: "\[test\]"
```
**Special characters escaped**: `. ^ $ * + ? { } [ ] \ | ( )`
### wildcard_to_regex
Converts simple wildcard patterns to regex patterns for flexible file matching.
**Conversion rules:**
- `*` becomes `.*` (matches any sequence of characters)
- Literal text is escaped (special regex characters are escaped)
- Patterns without `*` match as substrings anywhere
```v
import incubaid.herolib.core.texttools.regext
// Match files ending with .txt
pattern1 := regext.wildcard_to_regex("*.txt")
// Result: ".*\.txt"
// Match anything starting with test
pattern2 := regext.wildcard_to_regex("test*")
// Result: "test.*"
// Match anything containing 'config' (no wildcard)
pattern3 := regext.wildcard_to_regex("config")
// Result: ".*config.*"
// Complex pattern with special chars
pattern4 := regext.wildcard_to_regex("src/*.v")
// Result: "src/.*\.v"
// Multiple wildcards
pattern5 := regext.wildcard_to_regex("*test*file*")
// Result: ".*test.*file.*"
```
## Regex Group Finders
### find_sid
Extracts unique `sid` values from a given text. A `sid` is identified by the pattern `sid:XXXXXX`, where `XXXXXX` can be alphanumeric characters.
```v
import incubaid.herolib.core.texttools.regext
text := `
!!action.something sid:aa733
sid:aa733
...sid:aa733 ss
...sid:rrrrrr ss
sid:997
sid:s d
sid:s_d
`
r := regext.find_sid(text)
// Result: ['aa733', 'aa733', 'aa733', '997']
```
### find_simple_vars
Extracts simple variable names enclosed in curly braces, e.g., `{var_name}`, from a given text. Variable names can contain letters, numbers, and underscores.
```v
import incubaid.herolib.core.texttools.regext
text := `
!!action.something {sid}
sid:aa733
{a}
...sid:rrrrrr ss {a_sdsdsdsd_e__f_g}
sid:997
sid:s d
sid:s_d
`
r := regext.find_simple_vars(text)
// Result: ['sid', 'a', 'a_sdsdsdsd_e__f_g']
```
## regex replacer
Tool to flexibly replace elements in file(s) or text.
next example does it for
```golang
import incubaid.herolib.core.texttools.regext
text := '
@@ -51,7 +146,3 @@ mut text_out2 := ri.replace(text: text, dedent: true) or { panic(err) }
ri.replace_in_dir(path:"/tmp/mypath",extensions:["md"])!
```
```

44
lib/core/texttools/regext/regex_convert.v Normal file
View File

@@ -0,0 +1,44 @@
module regext
// escape_regex_chars escapes special regex metacharacters in a string
// This makes a literal string safe to use in regex patterns.
// Examples:
// "file.txt" -> "file\.txt"
// "a[123]" -> "a\[123\]"
pub fn escape_regex_chars(s string) string {
mut result := ''
for ch in s {
match ch {
`.`, `^`, `$`, `*`, `+`, `?`, `{`, `}`, `[`, `]`, `\\`, `|`, `(`, `)` {
result += '\\'
}
else {}
}
result += ch.ascii_str()
}
return result
}
// wildcard_to_regex converts a wildcard pattern (e.g., "*.txt") to a regex pattern.
// This function does not add implicit ^ and $ anchors, allowing for substring matches.
fn wildcard_to_regex(wildcard_pattern string) string {
mut regex_pattern := ''
for i, r in wildcard_pattern.runes() {
match r {
`*` {
regex_pattern += '.*'
}
`?` {
regex_pattern += '.'
}
`.`, `+`, `(`, `)`, `[`, `]`, `{`, `}`, `^`, `$`, `\\`, `|` {
// Escape regex special characters
regex_pattern += '\\' + r.str()
}
else {
regex_pattern += r.str()
}
}
}
return regex_pattern
}

88
lib/core/texttools/regext/regex_convert_test.v Normal file
View File

@@ -0,0 +1,88 @@
module regext
fn test_escape_regex_chars_special_chars() {
assert escape_regex_chars('.') == '\\.'
assert escape_regex_chars('^') == '\\^'
assert escape_regex_chars('$') == '\\$'
assert escape_regex_chars('*') == '\\*'
assert escape_regex_chars('+') == '\\+'
assert escape_regex_chars('?') == '\\?'
assert escape_regex_chars('{') == '\\{'
assert escape_regex_chars('}') == '\\}'
assert escape_regex_chars('[') == '\\['
assert escape_regex_chars(']') == '\\]'
assert escape_regex_chars('\\') == '\\\\'
assert escape_regex_chars('|') == '\\|'
assert escape_regex_chars('(') == '\\('
assert escape_regex_chars(')') == '\\)'
}
fn test_escape_regex_chars_normal_chars() {
assert escape_regex_chars('a') == 'a'
assert escape_regex_chars('1') == '1'
assert escape_regex_chars('hello') == 'hello'
assert escape_regex_chars('test_123') == 'test_123'
}
fn test_escape_regex_chars_mixed() {
assert escape_regex_chars('file.txt') == 'file\\.txt'
assert escape_regex_chars('test[1]') == 'test\\[1\\]'
assert escape_regex_chars('a.b*c') == 'a\\.b\\*c'
}
fn test_escape_regex_chars_empty() {
assert escape_regex_chars('') == ''
}
fn test_wildcard_to_regex_no_wildcard() {
// Pattern without wildcards returns substring matcher
assert wildcard_to_regex('config') == '.*config.*'
assert wildcard_to_regex('test.txt') == '.*test\\.txt.*'
assert wildcard_to_regex('hello') == '.*hello.*'
}
fn test_wildcard_to_regex_start_wildcard() {
// Pattern starting with *
assert wildcard_to_regex('*.txt') == '.*\\.txt'
assert wildcard_to_regex('*.v') == '.*\\.v'
assert wildcard_to_regex('*.log') == '.*\\.log'
}
fn test_wildcard_to_regex_end_wildcard() {
// Pattern ending with *
assert wildcard_to_regex('test*') == 'test.*'
assert wildcard_to_regex('log*') == 'log.*'
assert wildcard_to_regex('file_*') == 'file_.*'
}
fn test_wildcard_to_regex_middle_wildcard() {
// Pattern with * in the middle
assert wildcard_to_regex('test*file') == 'test.*file'
assert wildcard_to_regex('src*main.v') == 'src.*main\\.v'
}
fn test_wildcard_to_regex_multiple_wildcards() {
// Pattern with multiple wildcards
assert wildcard_to_regex('*test*') == '.*test.*'
assert wildcard_to_regex('*src*.v') == '.*src.*\\.v'
assert wildcard_to_regex('*a*b*c*') == '.*a.*b.*c.*'
}
fn test_wildcard_to_regex_only_wildcard() {
// Pattern with only wildcard(s)
assert wildcard_to_regex('*') == '.*'
assert wildcard_to_regex('**') == '.*.*'
}
fn test_wildcard_to_regex_special_chars_in_pattern() {
// Patterns containing special regex characters should be escaped
assert wildcard_to_regex('[test]') == '.*\\[test\\].*'
assert wildcard_to_regex('test.file') == '.*test\\.file.*'
assert wildcard_to_regex('(test)') == '.*\\(test\\).*'
}
fn test_wildcard_to_regex_edge_cases() {
assert wildcard_to_regex('') == '.*.*'
assert wildcard_to_regex('a') == '.*a.*'
assert wildcard_to_regex('.') == '.*\\..*'
}

1
lib/data/paramsparser/readme.md
View File

@@ -168,6 +168,7 @@ println(map_representation["key1"]) // Output: value1
Combine two `Params` objects, with values from the merged object overriding existing keys.
```v
mut params1 := paramsparser.new("color:red size:small")!
params2 := paramsparser.new("size:large material:wood")!

64
lib/develop/codewalker/README.md
View File

@@ -1,64 +0,0 @@
# CodeWalker Module
The CodeWalker module provides functionality to walk through directories and create a map of files with their content. It's particularly useful for processing code directories while respecting gitignore patterns.
## Features
- Walk through directories recursively
- Respect gitignore patterns to exclude files
- Store file content in memory
- Export files back to a directory structure
## Usage
```v
import incubaid.herolib.lib.lang.codewalker
mut cw := codewalker.new('/tmp/adir')!
// Get content of a specific file
content := cw.filemap.get('path/to/file.txt')!
// return output again
cw.filemap.content()
// Export all files to a destination directory
cw.filemap.export('/tmp/exported_files')!
```
### format of filemap
## full files
```
text before will be ignored
===FILE:filename===
code
===FILE:filename===
code
===END===
text behind will be ignored
```
## files with changes
```
text before will be ignored
===FILECHANGE:filename===
code
===FILECHANGE:filename===
code
===END===
text behind will be ignored
```
FILECHANGE and FILE can be mixed, in FILE it means we have full content otherwise only changed content e.g. a method or s struct and then we need to use morph to change it

219
lib/develop/codewalker/codewalker.v
View File

@@ -1,219 +0,0 @@
module codewalker
import incubaid.herolib.core.pathlib
pub struct CodeWalker {
pub mut:
ignorematcher IgnoreMatcher
errors []CWError
}
@[params]
pub struct FileMapArgs {
pub mut:
path string
content string
content_read bool = true // if we start from path, and this is on false then we don't read the content
}
// Public factory to parse the filemap-text format directly
pub fn (mut cw CodeWalker) parse(content string) !FileMap {
return cw.filemap_get_from_content(content)
}
pub fn (mut cw CodeWalker) filemap_get(args FileMapArgs) !FileMap {
if args.path != '' {
return cw.filemap_get_from_path(args.path, args.content_read)!
} else if args.content != '' {
return cw.filemap_get_from_content(args.content)!
} else {
return error('Either path or content must be provided to get FileMap')
}
}
// get the filemap from a path
fn (mut cw CodeWalker) filemap_get_from_path(path string, content_read bool) !FileMap {
mut dir := pathlib.get(path)
if !dir.exists() || !dir.is_dir() {
return error('Source directory "${path}" does not exist')
}
mut files := dir.list(ignore_default: false)!
mut fm := FileMap{
source: path
}
// collect ignore patterns from .gitignore and .heroignore files (recursively),
// and scope them to the directory where they were found
for mut p in files.paths {
if p.is_file() {
name := p.name()
if name == '.gitignore' || name == '.heroignore' {
content := p.read() or { '' }
if content != '' {
rel := p.path_relative(path) or { '' }
base_rel := if rel.contains('/') { rel.all_before_last('/') } else { '' }
cw.ignorematcher.add_content_with_base(base_rel, content)
}
}
}
}
for mut file in files.paths {
if file.is_file() {
name := file.name()
if name == '.gitignore' || name == '.heroignore' {
continue
}
relpath := file.path_relative(path)!
if cw.ignorematcher.is_ignored(relpath) {
continue
}
if content_read {
content := file.read()!
fm.content[relpath] = content
} else {
fm.content[relpath] = ''
}
}
}
return fm
}
// Parse a header line and return (kind, filename)
// kind: 'FILE' | 'FILECHANGE' | 'LEGACY' | 'END'
fn (mut cw CodeWalker) parse_header(line string, linenr int) !(string, string) {
if line == '===END===' {
return 'END', ''
}
if line.starts_with('===FILE:') && line.ends_with('===') {
name := line.trim_left('=').trim_right('=').all_after(':').trim_space()
if name.len < 1 {
cw.error('Invalid filename, < 1 chars.', linenr, 'filename_get', true)!
}
return 'FILE', name
}
if line.starts_with('===FILECHANGE:') && line.ends_with('===') {
name := line.trim_left('=').trim_right('=').all_after(':').trim_space()
if name.len < 1 {
cw.error('Invalid filename, < 1 chars.', linenr, 'filename_get', true)!
}
return 'FILECHANGE', name
}
// Legacy header: ===filename===
if line.starts_with('===') && line.ends_with('===') {
name := line.trim('=').trim_space()
if name == 'END' {
return 'END', ''
}
if name.len < 1 {
cw.error('Invalid filename, < 1 chars.', linenr, 'filename_get', true)!
}
return 'LEGACY', name
}
return '', ''
}
fn (mut cw CodeWalker) error(msg string, linenr int, category string, fail bool) ! {
cw.errors << CWError{
message: msg
linenr: linenr
category: category
}
if fail {
return error(msg)
}
}
// internal function to get the filename
fn (mut cw CodeWalker) parse_filename_get(line string, linenr int) !string {
parts := line.split('===')
if parts.len < 2 {
cw.error('Invalid filename line: ${line}.', linenr, 'filename_get', true)!
}
mut name := parts[1].trim_space()
if name.len < 2 {
cw.error('Invalid filename, < 2 chars: ${name}.', linenr, 'filename_get', true)!
}
return name
}
enum ParseState {
start
in_block
}
// Parse filemap content string
fn (mut cw CodeWalker) filemap_get_from_content(content string) !FileMap {
mut fm := FileMap{}
mut current_kind := '' // 'FILE' | 'FILECHANGE' | 'LEGACY'
mut filename := ''
mut block := []string{}
mut had_any_block := false
mut linenr := 0
for line in content.split_into_lines() {
linenr += 1
line2 := line.trim_space()
kind, name := cw.parse_header(line2, linenr)!
if kind == 'END' {
if filename == '' {
if had_any_block {
cw.error("Filename 'END' is reserved.", linenr, 'parse', true)!
} else {
cw.error('END found at start, not good.', linenr, 'parse', true)!
}
} else {
if current_kind == 'FILE' || current_kind == 'LEGACY' {
fm.content[filename] = block.join_lines()
} else if current_kind == 'FILECHANGE' {
fm.content_change[filename] = block.join_lines()
}
filename = ''
block = []string{}
current_kind = ''
}
continue
}
if kind in ['FILE', 'FILECHANGE', 'LEGACY'] {
// starting a new block header
if filename != '' {
if current_kind == 'FILE' || current_kind == 'LEGACY' {
fm.content[filename] = block.join_lines()
} else if current_kind == 'FILECHANGE' {
fm.content_change[filename] = block.join_lines()
}
}
filename = name
current_kind = kind
block = []string{}
had_any_block = true
continue
}
// Non-header line
if filename == '' {
if line2.len > 0 {
cw.error("Unexpected content before first file block: '${line}'.", linenr,
'parse', false)!
}
} else {
block << line
}
}
// EOF: flush current block if any
if filename != '' {
if current_kind == 'FILE' || current_kind == 'LEGACY' {
fm.content[filename] = block.join_lines()
} else if current_kind == 'FILECHANGE' {
fm.content_change[filename] = block.join_lines()
}
}
return fm
}

253
lib/develop/codewalker/codewalker_test.v
View File

@@ -1,253 +0,0 @@
module codewalker
import os
import incubaid.herolib.core.pathlib
fn test_parse_basic() {
mut cw := new(CodeWalkerArgs{})!
test_content := '===FILE:file1.txt===\nline1\nline2\n===END==='
fm := cw.parse(test_content)!
assert fm.content.len == 1
assert fm.content['file1.txt'] == 'line1\nline2'
}
fn test_parse_multiple_files() {
mut cw := new(CodeWalkerArgs{})!
test_content := '===FILE:file1.txt===\nline1\n===FILE:file2.txt===\nlineA\nlineB\n===END==='
fm := cw.parse(test_content)!
assert fm.content.len == 2
assert fm.content['file1.txt'] == 'line1'
assert fm.content['file2.txt'] == 'lineA\nlineB'
}
fn test_parse_empty_file_block() {
mut cw := new(CodeWalkerArgs{})!
test_content := '===FILE:empty.txt===\n===END==='
fm := cw.parse(test_content)!
assert fm.content.len == 1
assert fm.content['empty.txt'] == ''
}
fn test_parse_consecutive_end_and_file() {
mut cw := new(CodeWalkerArgs{})!
test_content := '===FILE:file1.txt===\ncontent1\n===END===\n===FILE:file2.txt===\ncontent2\n===END==='
fm := cw.parse(test_content)!
assert fm.content.len == 2
assert fm.content['file1.txt'] == 'content1'
assert fm.content['file2.txt'] == 'content2'
}
fn test_parse_content_before_first_file_block() {
mut cw := new(CodeWalkerArgs{})!
test_content := 'unexpected content\n===FILE:file1.txt===\ncontent\n===END==='
// This should ideally log an error but still parse the file
fm := cw.parse(test_content)!
assert fm.content.len == 1
assert fm.content['file1.txt'] == 'content'
assert cw.errors.len > 0
assert cw.errors[0].message.contains('Unexpected content before first file block')
}
fn test_parse_content_after_end() {
mut cw := new(CodeWalkerArgs{})!
test_content := '===FILE:file1.txt===\ncontent\n===END===\nmore unexpected content'
// Implementation chooses to ignore content after END but return parsed content
fm := cw.parse(test_content)!
assert fm.content.len == 1
assert fm.content['file1.txt'] == 'content'
}
fn test_parse_invalid_filename_line() {
mut cw := new(CodeWalkerArgs{})!
test_content := '======\ncontent\n===END==='
cw.parse(test_content) or {
assert err.msg().contains('Invalid filename, < 1 chars')
return
}
assert false // Should have errored
}
fn test_parse_file_ending_without_end() {
mut cw := new(CodeWalkerArgs{})!
test_content := '===FILE:file1.txt===\nline1\nline2'
fm := cw.parse(test_content)!
assert fm.content.len == 1
assert fm.content['file1.txt'] == 'line1\nline2'
}
fn test_parse_empty_content() {
mut cw := new(CodeWalkerArgs{})!
test_content := ''
fm := cw.parse(test_content)!
assert fm.content.len == 0
}
fn test_parse_only_end_at_start() {
mut cw := new(CodeWalkerArgs{})!
test_content := '===END==='
cw.parse(test_content) or {
assert err.msg().contains('END found at start, not good.')
return
}
assert false // Should have errored
}
fn test_parse_mixed_file_and_filechange() {
mut cw2 := new(CodeWalkerArgs{})!
test_content2 := '===FILE:file.txt===\nfull\n===FILECHANGE:file.txt===\npartial\n===END==='
fm2 := cw2.parse(test_content2)!
assert fm2.content.len == 1
assert fm2.content_change.len == 1
assert fm2.content['file.txt'] == 'full'
assert fm2.content_change['file.txt'] == 'partial'
}
fn test_parse_empty_block_between_files() {
mut cw := new(CodeWalkerArgs{})!
test_content := '===FILE:file1.txt===\ncontent1\n===FILE:file2.txt===\n===END===\n===FILE:file3.txt===\ncontent3\n===END==='
fm := cw.parse(test_content)!
assert fm.content.len == 3
assert fm.content['file1.txt'] == 'content1'
assert fm.content['file2.txt'] == ''
assert fm.content['file3.txt'] == 'content3'
}
fn test_parse_multiple_empty_blocks() {
mut cw := new(CodeWalkerArgs{})!
test_content := '===FILE:file1.txt===\n===END===\n===FILE:file2.txt===\n===END===\n===FILE:file3.txt===\ncontent3\n===END==='
fm := cw.parse(test_content)!
assert fm.content.len == 3
assert fm.content['file1.txt'] == ''
assert fm.content['file2.txt'] == ''
assert fm.content['file3.txt'] == 'content3'
}
fn test_parse_filename_end_reserved() {
mut cw := new(CodeWalkerArgs{})!
// Legacy header 'END' used as filename should error when used as header for new block
test_content := '===file1.txt===\ncontent1\n===END===\n===END===\ncontent2\n===END==='
cw.parse(test_content) or {
assert err.msg().contains("Filename 'END' is reserved.")
return
}
assert false // Should have errored
}
fn test_filemap_export_and_write() ! {
// Setup temp dir
mut tmpdir := pathlib.get_dir(
path: os.join_path(os.temp_dir(), 'cw_test')
create: true
empty: true
)!
defer {
tmpdir.delete() or {}
}
// Build a FileMap
mut fm := FileMap{
source: tmpdir.path
}
fm.set('a/b.txt', 'hello')
fm.set('c.txt', 'world')
// Export to new dir
mut dest := pathlib.get_dir(
path: os.join_path(os.temp_dir(), 'cw_out')
create: true
empty: true
)!
defer {
dest.delete() or {}
}
fm.export(dest.path)!
mut f1 := pathlib.get_file(path: os.join_path(dest.path, 'a/b.txt'))!
mut f2 := pathlib.get_file(path: os.join_path(dest.path, 'c.txt'))!
assert f1.read()! == 'hello'
assert f2.read()! == 'world'
// Overwrite via write()
fm.set('a/b.txt', 'hello2')
fm.write(dest.path)!
assert f1.read()! == 'hello2'
}
fn test_filemap_content_roundtrip() {
mut fm := FileMap{}
fm.set('x.txt', 'X')
fm.content_change['y.txt'] = 'Y'
txt := fm.content()
assert txt.contains('===FILE:x.txt===')
assert txt.contains('===FILECHANGE:y.txt===')
assert txt.contains('===END===')
}
fn test_ignore_level_scoped() ! {
// create temp dir structure
mut root := pathlib.get_dir(
path: os.join_path(os.temp_dir(), 'cw_ign_lvl')
create: true
empty: true
)!
defer { root.delete() or {} }
// subdir with its own ignore
mut sub := pathlib.get_dir(path: os.join_path(root.path, 'sub'), create: true)!
mut hero := pathlib.get_file(path: os.join_path(sub.path, '.heroignore'), create: true)!
hero.write('dist/\n')!
// files under sub/dist should be ignored
mut dist := pathlib.get_dir(path: os.join_path(sub.path, 'dist'), create: true)!
mut a1 := pathlib.get_file(path: os.join_path(dist.path, 'a.txt'), create: true)!
a1.write('A')!
// sibling sub2 with a dist, should NOT be ignored by sub's .heroignore
mut sub2 := pathlib.get_dir(path: os.join_path(root.path, 'sub2'), create: true)!
mut dist2 := pathlib.get_dir(path: os.join_path(sub2.path, 'dist'), create: true)!
mut b1 := pathlib.get_file(path: os.join_path(dist2.path, 'b.txt'), create: true)!
b1.write('B')!
// a normal file under sub should be included
mut okf := pathlib.get_file(path: os.join_path(sub.path, 'ok.txt'), create: true)!
okf.write('OK')!
mut cw := new(CodeWalkerArgs{})!
mut fm := cw.filemap_get(path: root.path)!
// sub/dist/a.txt should be ignored
assert 'sub/dist/a.txt' !in fm.content.keys()
// sub/ok.txt should be included
assert fm.content['sub/ok.txt'] == 'OK'
// sub2/dist/b.txt should be included (since .heroignore is level-scoped)
assert fm.content['sub2/dist/b.txt'] == 'B'
}
fn test_ignore_level_scoped_gitignore() ! {
mut root := pathlib.get_dir(
path: os.join_path(os.temp_dir(), 'cw_ign_git')
create: true
empty: true
)!
defer { root.delete() or {} }
// root has .gitignore ignoring logs/
mut g := pathlib.get_file(path: os.join_path(root.path, '.gitignore'), create: true)!
g.write('logs/\n')!
// nested structure
mut svc := pathlib.get_dir(path: os.join_path(root.path, 'svc'), create: true)!
// this logs/ should be ignored due to root .gitignore
mut logs := pathlib.get_dir(path: os.join_path(svc.path, 'logs'), create: true)!
mut out := pathlib.get_file(path: os.join_path(logs.path, 'out.txt'), create: true)!
out.write('ignored')!
// regular file should be included
mut appf := pathlib.get_file(path: os.join_path(svc.path, 'app.txt'), create: true)!
appf.write('app')!
mut cw := new(CodeWalkerArgs{})!
mut fm := cw.filemap_get(path: root.path)!
assert 'svc/logs/out.txt' !in fm.content.keys()
assert fm.content['svc/app.txt'] == 'app'
}
fn test_parse_filename_end_reserved_legacy() {
mut cw := new(CodeWalkerArgs{})!
// Legacy header 'END' used as filename should error when used as header for new block
test_content := '===file1.txt===\ncontent1\n===END===\n===END===\ncontent2\n===END==='
cw.parse(test_content) or {
assert err.msg().contains("Filename 'END' is reserved.")
return
}
assert false // Should have errored
}

12
lib/develop/codewalker/factory.v
View File

@@ -1,12 +0,0 @@
module codewalker
@[params]
pub struct CodeWalkerArgs {
// No fields required for now; kept for API stability
}
pub fn new(args CodeWalkerArgs) !CodeWalker {
mut cw := CodeWalker{}
cw.ignorematcher = gitignore_matcher_new()
return cw
}

161
lib/develop/codewalker/ignore.v
View File

@@ -1,161 +0,0 @@
module codewalker
// A minimal gitignore-like matcher used by CodeWalker
// Supports:
// - Directory patterns ending with '/': ignores any path that has this segment prefix
// - Extension patterns like '*.pyc' or '*.<ext>'
// - Simple substrings and '*' wildcards
// - Lines starting with '#' are comments; empty lines ignored
// No negation support for simplicity
const default_gitignore = '
.git/
.svn/
.hg/
.bzr/
node_modules/
__pycache__/
*.py[cod]
*.so
.Python
build/
develop-eggs/
downloads/
eggs/
.eggs/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
.env
.venv
venv/
.tox/
.nox/
.coverage
.coveragerc
coverage.xml
*.cover
*.gem
*.pyc
.cache
.pytest_cache/
.mypy_cache/
.hypothesis/
.DS_Store
Thumbs.db
*.tmp
*.temp
*.log
'
struct IgnoreRule {
base string // relative dir from source root where the ignore file lives ('' means global)
pattern string
}
pub struct IgnoreMatcher {
pub mut:
rules []IgnoreRule
}
pub fn gitignore_matcher_new() IgnoreMatcher {
mut m := IgnoreMatcher{}
m.add_content(default_gitignore)
return m
}
// Add raw .gitignore-style content as global (root-scoped) rules
pub fn (mut m IgnoreMatcher) add_content(content string) {
m.add_content_with_base('', content)
}
// Add raw .gitignore/.heroignore-style content scoped to base_rel
pub fn (mut m IgnoreMatcher) add_content_with_base(base_rel string, content string) {
mut base := base_rel.replace('\\', '/').trim('/').to_lower()
for raw_line in content.split_into_lines() {
mut line := raw_line.trim_space()
if line.len == 0 || line.starts_with('#') {
continue
}
m.rules << IgnoreRule{
base: base
pattern: line
}
}
}
// Very simple glob/substring-based matching with directory scoping
pub fn (m IgnoreMatcher) is_ignored(relpath string) bool {
mut path := relpath.replace('\\', '/').trim_left('/')
path_low := path.to_lower()
for rule in m.rules {
mut pat := rule.pattern.replace('\\', '/').trim_space()
if pat == '' {
continue
}
// Determine subpath relative to base
mut sub := path_low
if rule.base != '' {
base := rule.base
if sub == base {
// path equals the base dir; ignore rules apply to entries under base, not the base itself
continue
}
if sub.starts_with(base + '/') {
sub = sub[(base.len + 1)..]
} else {
continue // rule not applicable for this path
}
}
// Directory pattern (relative to base)
if pat.ends_with('/') {
mut dirpat := pat.trim_right('/')
dirpat = dirpat.trim_left('/').to_lower()
if sub == dirpat || sub.starts_with(dirpat + '/') || sub.contains('/' + dirpat + '/') {
return true
}
continue
}
// Extension pattern *.ext
if pat.starts_with('*.') {
ext := pat.all_after_last('.').to_lower()
if sub.ends_with('.' + ext) {
return true
}
continue
}
// Simple wildcard * anywhere -> sequential contains match
if pat.contains('*') {
mut parts := pat.to_lower().split('*')
mut idx := 0
mut ok := true
for part in parts {
if part == '' {
continue
}
pos := sub.index_after(part, idx) or { -1 }
if pos == -1 {
ok = false
break
}
idx = pos + part.len
}
if ok {
return true
}
continue
}
// Fallback: substring match (case-insensitive) on subpath
if sub.contains(pat.to_lower()) {
return true
}
}
return false
}

16
lib/develop/codewalker/model.v
View File

@@ -1,16 +0,0 @@
module codewalker
pub struct CWError {
pub:
message string
linenr int
category string
}
pub struct FMError {
pub:
message string
linenr int // is optional
category string
filename string
}

46
lib/develop/heroprompt/heroprompt_workspace.v
View File

@@ -4,7 +4,7 @@ import rand
import time
import os
import incubaid.herolib.core.pathlib
import incubaid.herolib.develop.codewalker
import incubaid.herolib.ai.filemap
// Selection API
@[params]
@@ -222,14 +222,23 @@ pub:
}
pub fn (wsp &Workspace) list_dir(rel_path string) ![]ListItem {
// Create an ignore matcher with default patterns
ignore_matcher := codewalker.gitignore_matcher_new()
items := codewalker.list_directory_filtered(wsp.base_path, rel_path, &ignore_matcher)!
// Use pathlib to list directory with default ignore patterns
full_path := if rel_path.len == 0 {
wsp.base_path
} else {
os.join_path(wsp.base_path, rel_path)
}
mut dir := pathlib.get(full_path)
// List with default ignore patterns (files starting with . and _)
mut list_result := dir.list(recursive: false, ignore_default: true)!
mut out := []ListItem{}
for item in items {
for mut path_item in list_result.paths {
typ := if path_item.is_dir() { 'dir' } else { 'file' }
out << ListItem{
name: item.name
typ: item.typ
name: os.base(path_item.path)
typ: typ
}
}
return out
@@ -268,11 +277,10 @@ fn (wsp Workspace) build_file_content() !string {
}
}
}
// files under selected directories, using CodeWalker for filtered traversal
// files under selected directories, using filemap for filtered traversal
for ch in wsp.children {
if ch.path.cat == .dir && ch.include_tree {
mut cw := codewalker.new(codewalker.CodeWalkerArgs{})!
mut fm := cw.filemap_get(path: ch.path.path)!
mut fm := filemap.filemap(path: ch.path.path)!
for rel, fc in fm.content {
if content.len > 0 {
content += '\n\n'
@@ -303,7 +311,7 @@ fn (wsp Workspace) build_user_instructions(text string) string {
}
// build_file_map creates a complete file map with base path and metadata
fn (wsp Workspace) build_file_map() string {
fn (wsp Workspace) build_file_map() !string {
mut file_map := ''
// roots are selected directories
mut roots := []HeropromptChild{}
@@ -342,13 +350,15 @@ fn (wsp Workspace) build_file_map() string {
// files under dirs (only when roots present)
if roots.len > 0 {
for r in roots {
for f in codewalker.list_files_recursive(r.path.path) {
mut dir := pathlib.get(r.path.path)
mut file_list := dir.list(recursive: true, files_only: true)!
for mut f in file_list.paths {
total_files++
ext := get_file_extension(os.base(f))
ext := get_file_extension(os.base(f.path))
if ext.len > 0 {
file_extensions[ext] = file_extensions[ext] + 1
}
total_content_length += (os.read_file(f) or { '' }).len
total_content_length += (os.read_file(f.path) or { '' }).len
}
}
}
@@ -386,16 +396,16 @@ fn (wsp Workspace) build_file_map() string {
for r in roots {
root_paths << r.path.path
}
file_map += codewalker.build_file_tree_fs(root_paths, '')
file_map += build_file_tree_fs(root_paths, '')
}
// If there are only standalone selected files (no selected dirs),
// build a minimal tree via codewalker relative to the workspace base.
// build a minimal tree relative to the workspace base.
if files_only.len > 0 && roots.len == 0 {
mut paths := []string{}
for fo in files_only {
paths << fo.path.path
}
file_map += codewalker.build_selected_tree(paths, wsp.base_path)
file_map += build_selected_tree(paths, wsp.base_path)
} else if files_only.len > 0 && roots.len > 0 {
// Keep listing absolute paths for standalone files when directories are also selected.
for fo in files_only {
@@ -413,7 +423,7 @@ pub mut:
pub fn (wsp Workspace) prompt(args WorkspacePrompt) string {
user_instructions := wsp.build_user_instructions(args.text)
file_map := wsp.build_file_map()
file_map := wsp.build_file_map() or { '(Error building file map)' }
file_contents := wsp.build_file_content() or { '(Error building file contents)' }
prompt := HeropromptTmpPrompt{
user_instructions: user_instructions

95
lib/develop/codewalker/tree.v → lib/develop/heroprompt/tree.v
View File

@@ -1,4 +1,4 @@
module codewalker
module heroprompt
import os
@@ -82,99 +82,6 @@ pub:
typ string
}
// list_directory lists the contents of a directory.
// - base_path: workspace base path
// - rel_path: relative path from base (or absolute path)
// Returns a list of DirItem with name and type (file/directory).
pub fn list_directory(base_path string, rel_path string) ![]DirItem {
dir := resolve_path(base_path, rel_path)
if dir.len == 0 {
return error('base_path not set')
}
entries := os.ls(dir) or { return error('cannot list directory') }
mut out := []DirItem{}
for e in entries {
full := os.join_path(dir, e)
if os.is_dir(full) {
out << DirItem{
name: e
typ: 'directory'
}
} else if os.is_file(full) {
out << DirItem{
name: e
typ: 'file'
}
}
}
return out
}
// list_directory_filtered lists the contents of a directory with ignore filtering applied.
// - base_path: workspace base path
// - rel_path: relative path from base (or absolute path)
// - ignore_matcher: IgnoreMatcher to filter out ignored files/directories
// Returns a list of DirItem with name and type (file/directory), filtered by ignore patterns.
pub fn list_directory_filtered(base_path string, rel_path string, ignore_matcher &IgnoreMatcher) ![]DirItem {
dir := resolve_path(base_path, rel_path)
if dir.len == 0 {
return error('base_path not set')
}
entries := os.ls(dir) or { return error('cannot list directory') }
mut out := []DirItem{}
for e in entries {
full := os.join_path(dir, e)
// Calculate relative path from base_path for ignore checking
mut check_path := if rel_path.len > 0 {
if rel_path.ends_with('/') { rel_path + e } else { rel_path + '/' + e }
} else {
e
}
// For directories, also check with trailing slash
is_directory := os.is_dir(full)
mut should_ignore := ignore_matcher.is_ignored(check_path)
if is_directory && !should_ignore {
// Also check directory pattern with trailing slash
should_ignore = ignore_matcher.is_ignored(check_path + '/')
}
// Check if this entry should be ignored
if should_ignore {
continue
}
if is_directory {
out << DirItem{
name: e
typ: 'directory'
}
} else if os.is_file(full) {
out << DirItem{
name: e
typ: 'file'
}
}
}
return out
}
// list_files_recursive recursively lists all files in a directory
pub fn list_files_recursive(root string) []string {
mut out := []string{}
entries := os.ls(root) or { return out }
for e in entries {
fp := os.join_path(root, e)
if os.is_dir(fp) {
out << list_files_recursive(fp)
} else if os.is_file(fp) {
out << fp
}
}
return out
}
// build_file_tree_fs builds a file system tree for given root directories
pub fn build_file_tree_fs(roots []string, prefix string) string {
mut out := ''

1
lib/hero/heromodels/prd_test.v
View File

@@ -188,7 +188,6 @@ fn test_prd_list() ! {
mut mydb := db.new_test()!
// Clear the test database to ensure clean state
mydb.redis.flushdb()!
mut db_prd := DBPrd{
db: &mydb
}

54
lib/threefold/models_to_move/core/comment.v
View File

@@ -1,54 +0,0 @@
module core
// Comment represents a generic commenting functionality that can be associated with any other model
// It supports threaded conversations through parent_comment_id
@[heap]
pub struct Comment {
pub mut:
id u32 // Unique comment ID
user_id u32 // ID of the user who posted the comment (indexed)
content string // The text content of the comment
parent_comment_id ?u32 // Optional parent comment ID for threaded comments
created_at u64 // Creation timestamp
updated_at u64 // Last update timestamp
}
// new creates a new Comment with default values
pub fn Comment.new() Comment {
return Comment{
id: 0
user_id: 0
content: ''
parent_comment_id: none
created_at: 0
updated_at: 0
}
}
// user_id sets the user ID for the comment (builder pattern)
pub fn (mut c Comment) user_id(id u32) Comment {
c.user_id = id
return c
}
// content sets the content for the comment (builder pattern)
pub fn (mut c Comment) content(text string) Comment {
c.content = text
return c
}
// parent_comment_id sets the parent comment ID for threaded comments (builder pattern)
pub fn (mut c Comment) parent_comment_id(parent_id ?u32) Comment {
c.parent_comment_id = parent_id
return c
}
// is_top_level returns true if this is a top-level comment (no parent)
pub fn (c Comment) is_top_level() bool {
return c.parent_comment_id == none
}
// is_reply returns true if this is a reply to another comment
pub fn (c Comment) is_reply() bool {
return c.parent_comment_id != none
}