MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

...

Browse Source
This commit is contained in:
despiegk
2025-09-15 06:19:47 +02:00
parent 07ca315299
commit 1f5c75dcd5
8 changed files with 244 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
examples/hero/herofs/fs_dir_test.vsh Executable file
View File

@@ -0,0 +1,143 @@
#!/usr/bin/env -S v -n -w -cg -gc none -cc tcc -d use_openssl -enable-globals run
import freeflowuniverse.herolib.hero.herofs
import freeflowuniverse.herolib.hero.db
fn main() {
println('Testing FsDir functionality...')
// Initialize the HeroFS factory
mut fs_factory := herofs.new()!
println('HeroFS factory initialized')
// Create a new filesystem (required for FsDir)
mut my_fs := fs_factory.fs.new(
name: 'test_filesystem'
description: 'Filesystem for testing FsDir functionality'
quota_bytes: 1024 * 1024 * 1024 // 1GB quota
)!
// Save the filesystem to get an ID
fs_id := fs_factory.fs.set(my_fs)!
println('Created test filesystem with ID: ${fs_id}')
// Create root directory
mut root_dir := fs_factory.fs_dir.new(
name: 'root'
fs_id: fs_id
parent_id: 0 // Root has no parent
description: 'Root directory for testing'
)!
// Save the root directory
root_dir_id := fs_factory.fs_dir.set(root_dir)!
println('Created root directory with ID: ${root_dir_id}')
// Update the filesystem with the root directory ID
my_fs.root_dir_id = root_dir_id
fs_factory.fs.set(my_fs)!
// Create test directories with various parameters
mut test_dir1 := fs_factory.fs_dir.new(
name: 'test_dir1'
fs_id: fs_id
parent_id: root_dir_id
description: 'First test directory'
)!
mut test_dir2 := fs_factory.fs_dir.new(
name: 'test_dir2'
fs_id: fs_id
parent_id: root_dir_id
description: 'Second test directory with tags'
tags: ['test', 'directory', 'example']
)!
mut test_dir3 := fs_factory.fs_dir.new(
name: 'test_dir3'
fs_id: fs_id
parent_id: root_dir_id
description: 'Third test directory with comments'
comments: [db.CommentArg{
text: 'This is a test comment'
author: 'test_user'
}]
)!
// Save the test directories
dir1_id := fs_factory.fs_dir.set(test_dir1)!
dir2_id := fs_factory.fs_dir.set(test_dir2)!
dir3_id := fs_factory.fs_dir.set(test_dir3)!
println('Created test directories:')
println('- ${test_dir1.name} with ID: ${dir1_id}')
println('- ${test_dir2.name} with ID: ${dir2_id}')
println('- ${test_dir3.name} with ID: ${dir3_id}')
// Test loading directories by ID
println('\nTesting directory loading...')
loaded_root_dir := fs_factory.fs_dir.get(root_dir_id)!
println('Loaded root directory: ${loaded_root_dir.name} (ID: ${loaded_root_dir.id})')
loaded_dir1 := fs_factory.fs_dir.get(dir1_id)!
println('Loaded test_dir1: ${loaded_dir1.name} (ID: ${loaded_dir1.id})')
println(' Description: ${loaded_dir1.description}')
loaded_dir2 := fs_factory.fs_dir.get(dir2_id)!
println('Loaded test_dir2: ${loaded_dir2.name} (ID: ${loaded_dir2.id})')
println(' Description: ${loaded_dir2.description}')
println(' Tags: ${loaded_dir2.tags}')
loaded_dir3 := fs_factory.fs_dir.get(dir3_id)!
println('Loaded test_dir3: ${loaded_dir3.name} (ID: ${loaded_dir3.id})')
println(' Description: ${loaded_dir3.description}')
// Verify that loaded directories match the original ones
println('\nVerifying data integrity...')
if loaded_root_dir.name == root_dir.name && loaded_root_dir.description == root_dir.description {
println('✓ Root directory data integrity verified')
} else {
println('✗ Root directory data integrity check failed')
}
if loaded_dir1.name == test_dir1.name && loaded_dir1.description == test_dir1.description {
println('✓ Test directory 1 data integrity verified')
} else {
println('✗ Test directory 1 data integrity check failed')
}
if loaded_dir2.name == test_dir2.name && loaded_dir2.description == test_dir2.description && loaded_dir2.tags == test_dir2.tags {
println('✓ Test directory 2 data integrity verified')
} else {
println('✗ Test directory 2 data integrity check failed')
}
if loaded_dir3.name == test_dir3.name && loaded_dir3.description == test_dir3.description {
println('✓ Test directory 3 data integrity verified')
} else {
println('✗ Test directory 3 data integrity check failed')
}
// Test exist method
println('\nTesting directory existence checks...')
exists := fs_factory.fs_dir.exist(root_dir_id)!
println('Root directory exists: ${exists}')
exists = fs_factory.fs_dir.exist(dir1_id)!
println('Test directory 1 exists: ${exists}')
exists = fs_factory.fs_dir.exist(dir2_id)!
println('Test directory 2 exists: ${exists}')
exists = fs_factory.fs_dir.exist(dir3_id)!
println('Test directory 3 exists: ${exists}')
// Test with non-existent ID
exists = fs_factory.fs_dir.exist(999999)!
println('Non-existent directory exists: ${exists}')
println('\nFsDir test completed successfully!')
}

29
lib/hero/herofs/factory.v
View File

@@ -5,34 +5,41 @@ import freeflowuniverse.herolib.hero.db
@[heap]
pub struct FsFactory {
pub mut:
fs DBFs
fs_blob DBFsBlob
fs DBFs
fs_blob DBFsBlob
fs_blob_membership DBFsBlobMembership
fs_dir DBFsDir
fs_file DBFsFile
fs_symlink DBFsSymlink
fs_dir DBFsDir
fs_file DBFsFile
fs_symlink DBFsSymlink
}
pub fn new() !FsFactory {
mut mydb := db.new()!
return FsFactory{
fs: DBFs{
mut f := FsFactory{
fs: DBFs{
db: &mydb
}
fs_blob: DBFsBlob{
fs_blob: DBFsBlob{
db: &mydb
}
fs_blob_membership: DBFsBlobMembership{
db: &mydb
}
fs_dir: DBFsDir{
fs_dir: DBFsDir{
db: &mydb
}
fs_file: DBFsFile{
fs_file: DBFsFile{
db: &mydb
}
fs_symlink: DBFsSymlink{
fs_symlink: DBFsSymlink{
db: &mydb
}
}
f.fs.factory = &f
f.fs_blob.factory = &f
f.fs_blob_membership.factory = &f
f.fs_dir.factory = &f
f.fs_file.factory = &f
f.fs_symlink.factory = &f
return f
}

3
lib/hero/herofs/fs.v
View File

@@ -23,7 +23,8 @@ pub mut:
pub struct DBFs {
pub mut:
db &db.DB @[skip; str: skip]
db &db.DB @[skip; str: skip]
factory &FsFactory = unsafe { nil } @[skip; str: skip]
}
pub fn (self Fs) type_name() string {

61
lib/hero/herofs/fs_blob.v
View File

@@ -1,11 +1,9 @@
module herofs
import time
import crypto.blake3
import freeflowuniverse.herolib.data.encoder
import freeflowuniverse.herolib.data.ourtime
import freeflowuniverse.herolib.hero.db
import freeflowuniverse.herolib.core.texttools
// FsBlob represents binary data up to 1MB
@[heap]
@@ -19,7 +17,8 @@ pub mut:
pub struct DBFsBlob {
pub mut:
db &db.DB @[skip; str: skip]
db &db.DB @[skip; str: skip]
factory &FsFactory = unsafe { nil } @[skip; str: skip]
}
pub fn (self FsBlob) type_name() string {
@@ -41,7 +40,7 @@ fn (mut self DBFsBlob) load(mut o FsBlob, mut e encoder.Decoder) ! {
@[params]
pub struct FsBlobArg {
pub mut:
data []u8 @[required]
data []u8 @[required]
}
pub fn (mut blob FsBlob) calculate_hash() {
@@ -64,10 +63,6 @@ pub fn (mut self DBFsBlob) new(args FsBlobArg) !FsBlob {
o.calculate_hash()
// Set base fields
o.name = args.name
o.description = args.description
o.tags = self.db.tags_get(args.tags)!
o.comments = self.db.comments_get(args.comments)!
o.updated_at = ourtime.now().unix()
return o
@@ -90,7 +85,6 @@ pub fn (mut self DBFsBlob) set(o FsBlob) !u32 {
return id
}
pub fn (mut self DBFsBlob) delete(id u32) ! {
// Get the blob to retrieve its hash
mut blob := self.get(id)!
@@ -102,10 +96,25 @@ pub fn (mut self DBFsBlob) delete(id u32) ! {
self.db.delete[FsBlob](id)!
}
pub fn (mut self DBFsBlob) delete_multi(ids []u32) ! {
for id in ids {
self.delete(id)!
}
}
pub fn (mut self DBFsBlob) exist(id u32) !bool {
return self.db.exists[FsBlob](id)!
}
pub fn (mut self DBFsBlob) exist_multi(ids []u32) !bool {
for id in ids {
if !self.exist(id)! {
return false
}
}
return true
}
pub fn (mut self DBFsBlob) get(id u32) !FsBlob {
mut o, data := self.db.get_data[FsBlob](id)!
mut e_decoder := encoder.decoder_new(data)
@@ -113,17 +122,24 @@ pub fn (mut self DBFsBlob) get(id u32) !FsBlob {
return o
}
pub fn (mut self DBFsBlob) get_by_hash(hash string) !FsBlob {
id_str := self.db.redis.hget('fsblob:hashes', hash)!
if id_str == '' {
return error('Blob with hash "${hash}" not found')
pub fn (mut self DBFsBlob) get_multi(id []u32) ![]FsBlob {
mut blobs := []FsBlob{}
for i in id {
blobs << self.get(i)!
}
return self.get(id_str.u32())!
return blobs
}
pub fn (mut self DBFsBlob) get_by_hash(hash string) !FsBlob {
if self.factory.fs_blob_membership.exist(hash)! {
o := self.factory.fs_blob_membership.get(hash) or { panic('bug') }
return self.get(o.blobid)!
}
return error('Blob with hash ${hash} not found')
}
pub fn (mut self DBFsBlob) exists_by_hash(hash string) !bool {
id_str := self.db.redis.hget('fsblob:hashes', hash)!
return id_str != ''
return self.factory.fs_blob_membership.exist(hash)
}
pub fn (blob FsBlob) verify_integrity() bool {
@@ -131,16 +147,7 @@ pub fn (blob FsBlob) verify_integrity() bool {
return hash.hex()[..48] == blob.hash
}
// verify checks the integrity of a blob by its ID or hash
// Returns true if the blob's data matches its stored hash
pub fn (mut self DBFsBlob) verify(id_or_hash string) !bool {
// Try to parse as ID first
if id_or_hash.is_int() {
blob := self.get(id_or_hash.int().u32())!
return blob.verify_integrity()
}
// Otherwise treat as hash
blob := self.get_by_hash(id_or_hash)!
pub fn (mut self DBFsBlob) verify(hash string) !bool {
blob := self.get_by_hash(hash)!
return blob.verify_integrity()
}

65
lib/hero/herofs/fs_blob_membership.v
View File

@@ -1,24 +1,21 @@
module herofs
import time
import crypto.blake3
import freeflowuniverse.herolib.data.encoder
import freeflowuniverse.herolib.data.ourtime
import freeflowuniverse.herolib.hero.db
//FsBlobMembership represents membership of a blob in one or more filesystems, the key is the hash of the blob
// FsBlobMembership represents membership of a blob in one or more filesystems, the key is the hash of the blob
@[heap]
pub struct FsBlobMembership {
pub mut:
hash string // blake192 hash of content
fsid []u32 //list of fs ids where this blob is used
blobid u32 // id of the blob
hash string // blake192 hash of content
fsid []u32 // list of fs ids where this blob is used
blobid u32 // id of the blob
}
pub struct DBFsBlobMembership {
pub mut:
db &db.DB @[skip; str: skip]
db &db.DB @[skip; str: skip]
factory &FsFactory = unsafe { nil } @[skip; str: skip]
}
pub fn (self FsBlobMembership) type_name() string {
@@ -40,9 +37,9 @@ fn (mut self DBFsBlobMembership) load(mut o FsBlobMembership, mut e encoder.Deco
@[params]
pub struct FsBlobMembershipArg {
pub mut:
hash string @[required]
fsid []u32 @[required]
blobid u32 @[required]
hash string @[required]
fsid []u32 @[required]
blobid u32 @[required]
}
// get new blob membership, not from the DB
@@ -58,21 +55,21 @@ pub fn (mut self DBFsBlobMembership) new(args FsBlobMembershipArg) !FsBlobMember
pub fn (mut self DBFsBlobMembership) set(o FsBlobMembership) !string {
// Validate that the blob exists
blob_exists := self.db.fs_blob.exists(o.blobid)!
blob_exists := self.factory.fs_blob.exists(o.blobid)!
if !blob_exists {
return error('Blob with ID ${o.blobid} does not exist')
}
// Validate that all filesystems exist
for fs_id in o.fsid {
fs_exists := self.db.fs_file.exists(fs_id)!
fs_exists := self.factory.fs_file.exists(fs_id)!
if !fs_exists {
return error('Filesystem with ID ${fs_id} does not exist')
}
}
// Encode the object
mut e_encoder := encoder.encoder_new()
mut e_encoder := encoder.new()
o.dump(mut e_encoder)!
// Store using hash as key in the blob_membership hset
@@ -85,8 +82,7 @@ pub fn (mut self DBFsBlobMembership) delete(hash string) ! {
}
pub fn (mut self DBFsBlobMembership) exist(hash string) !bool {
result := self.db.redis.hexists('fs_blob_membership', hash)!
return result == 1
return self.db.redis.hexists('fs_blob_membership', hash)!
}
pub fn (mut self DBFsBlobMembership) get(hash string) !FsBlobMembership {
@@ -97,11 +93,11 @@ pub fn (mut self DBFsBlobMembership) get(hash string) !FsBlobMembership {
}
// Decode hex data back to bytes
data := data.bytes()
data2 := data.bytes()
// Create object and decode
mut o := FsBlobMembership{}
mut e_decoder := encoder.decoder_new(data)
mut e_decoder := encoder.decoder_new(data2)
self.load(mut o, mut e_decoder)!
return o
@@ -116,34 +112,31 @@ pub fn (mut self DBFsBlobMembership) add_filesystem(hash string, fs_id u32) !str
}
mut membership := self.get(hash)!
// Check if filesystem is already in the list
if fs_id !in membership.fsid {
membership.fsid << fs_id
}
return self.set(membership)!
}
// Remove a filesystem from an existing blob membership
pub fn (mut self DBFsBlobMembership) remove_filesystem(hash string, fs_id u32) !string {
mut membership := self.get(hash)!
// Remove filesystem from the list
membership.fsid = membership.fsid.filter(it != fs_id)
// If no filesystems left, delete the membership entirely
if membership.fsid.len == 0 {
self.delete(hash)!
return hash
}
return self.set(membership)!
}
// BlobList represents a simplified blob structure for listing purposes
pub struct BlobList {
pub mut:
@@ -158,18 +151,18 @@ pub fn (mut self DBFsBlobMembership) list(prefix string) ![]FsBlobMembership {
mut result := []FsBlobMembership{}
mut cursor := 0
mut count := 0
for {
if count >= 10000 {
break
}
// Use hscan with MATCH pattern and COUNT to iterate through the hash
new_cursor, values := self.db.redis.hscan('fs_blob_membership', cursor,
match: '${prefix}*',
new_cursor, values := self.db.redis.hscan('fs_blob_membership', cursor,
match: '${prefix}*'
count: 100
)!
// Process the returned field-value pairs
// hscan returns alternating field-value pairs, so we iterate by 2
mut i := 0
@@ -177,20 +170,20 @@ pub fn (mut self DBFsBlobMembership) list(prefix string) ![]FsBlobMembership {
hash := values[i]
// Skip the value (we don't need it since we'll get the object by hash)
i += 2
if hash.starts_with(prefix) {
result << self.get(hash)!
count++
}
}
// If cursor is "0", we've completed the full iteration
if new_cursor == '0' {
break
}
cursor = new_cursor.int()
}
return result
}

26
lib/hero/herofs/fs_dir.v
View File

@@ -12,8 +12,8 @@ import freeflowuniverse.herolib.hero.db
pub struct FsDir {
db.Base
pub mut:
fs_id u32 // Associated filesystem
parent_id u32 // Parent directory ID (0 for root)
fs_id u32 // Associated filesystem
parent_id u32 // Parent directory ID (0 for root)
directories []u32
files []u32
symlinks []u32
@@ -21,7 +21,8 @@ pub mut:
pub struct DBFsDir {
pub mut:
db &db.DB @[skip; str: skip]
db &db.DB @[skip; str: skip]
factory &FsFactory = unsafe { nil } @[skip; str: skip]
}
pub fn (self FsDir) type_name() string {
@@ -29,22 +30,21 @@ pub fn (self FsDir) type_name() string {
}
pub fn (self FsDir) dump(mut e encoder.Encoder) ! {
e.add_u32(self.fs_id)
e.add_u32(self.parent_id)
// Handle directories array
e.add_u16(u16(self.directories.len))
for dir_id in self.directories {
e.add_u32(dir_id)
}
// Handle files array
e.add_u16(u16(self.files.len))
for file_id in self.files {
e.add_u32(file_id)
}
// Handle symlinks array
e.add_u16(u16(self.symlinks.len))
for symlink_id in self.symlinks {
@@ -55,21 +55,21 @@ pub fn (self FsDir) dump(mut e encoder.Encoder) ! {
fn (mut self DBFsDir) load(mut o FsDir, mut e encoder.Decoder) ! {
o.fs_id = e.get_u32()!
o.parent_id = e.get_u32()!
// Load directories array
directories_count := e.get_u16()!
o.directories = []u32{cap: int(directories_count)}
for _ in 0 .. directories_count {
o.directories << e.get_u32()!
}
// Load files array
files_count := e.get_u16()!
o.files = []u32{cap: int(files_count)}
for _ in 0 .. files_count {
o.files << e.get_u32()!
}
// Load symlinks array
symlinks_count := e.get_u16()!
o.symlinks = []u32{cap: int(symlinks_count)}
@@ -121,14 +121,14 @@ pub fn (mut self DBFsDir) set(o FsDir) !u32 {
pub fn (mut self DBFsDir) delete(id u32) ! {
// Get the directory info before deleting
dir := self.get(id)!
// If has parent, remove from parent's directories list
if dir.parent_id > 0 {
mut parent_dir := self.db.directories.get(dir.parent_id) or {
mut parent_dir := self.factory.fs_dir.get(dir.parent_id) or {
return error('Parent directory with ID ${dir.parent_id} does not exist')
}
parent_dir.directories = parent_dir.directories.filter(it != id)
self.db.directories.set(parent_dir)!
self.factory.fs_dir.set(parent_dir)!
}
// Delete the directory itself
self.db.delete[FsDir](id)!

5
lib/hero/herofs/fs_file.v
View File

@@ -18,14 +18,15 @@ pub mut:
blobs []u32 // IDs of file content blobs
size_bytes u64
mime_type MimeType // MIME type as enum (MOVED FROM FsBlob)
checksum string // e.g., SHA256 checksum of the file
checksum string // e.g., SHA256 checksum of the file
accessed_at i64
metadata map[string]string // Custom metadata
}
pub struct DBFsFile {
pub mut:
db &db.DB @[skip; str: skip]
db &db.DB @[skip; str: skip]
factory &FsFactory = unsafe { nil } @[skip; str: skip]
}
pub fn (self FsFile) type_name() string {

3
lib/hero/herofs/fs_symlink.v
View File

@@ -26,7 +26,8 @@ pub enum SymlinkTargetType {
pub struct DBFsSymlink {
pub mut:
db &db.DB @[skip; str: skip]
db &db.DB @[skip; str: skip]
factory &FsFactory = unsafe { nil } @[skip; str: skip]
}
pub fn (self FsSymlink) type_name() string {