From 1f5c75dcd5a3dcbbaa7bdfa5f732d9820a57591a Mon Sep 17 00:00:00 2001 From: despiegk Date: Mon, 15 Sep 2025 06:19:47 +0200 Subject: [PATCH] ... --- examples/hero/herofs/fs_dir_test.vsh | 143 +++++++++++++++++++++++++++ lib/hero/herofs/factory.v | 29 +++--- lib/hero/herofs/fs.v | 3 +- lib/hero/herofs/fs_blob.v | 61 +++++++----- lib/hero/herofs/fs_blob_membership.v | 65 ++++++------ lib/hero/herofs/fs_dir.v | 26 ++--- lib/hero/herofs/fs_file.v | 5 +- lib/hero/herofs/fs_symlink.v | 3 +- 8 files changed, 244 insertions(+), 91 deletions(-) create mode 100755 examples/hero/herofs/fs_dir_test.vsh diff --git a/examples/hero/herofs/fs_dir_test.vsh b/examples/hero/herofs/fs_dir_test.vsh new file mode 100755 index 00000000..6bedfd7f --- /dev/null +++ b/examples/hero/herofs/fs_dir_test.vsh @@ -0,0 +1,143 @@ +#!/usr/bin/env -S v -n -w -cg -gc none -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.hero.herofs +import freeflowuniverse.herolib.hero.db + +fn main() { + println('Testing FsDir functionality...') + + // Initialize the HeroFS factory + mut fs_factory := herofs.new()! + println('HeroFS factory initialized') + + // Create a new filesystem (required for FsDir) + mut my_fs := fs_factory.fs.new( + name: 'test_filesystem' + description: 'Filesystem for testing FsDir functionality' + quota_bytes: 1024 * 1024 * 1024 // 1GB quota + )! + + // Save the filesystem to get an ID + fs_id := fs_factory.fs.set(my_fs)! + println('Created test filesystem with ID: ${fs_id}') + + // Create root directory + mut root_dir := fs_factory.fs_dir.new( + name: 'root' + fs_id: fs_id + parent_id: 0 // Root has no parent + description: 'Root directory for testing' + )! + + // Save the root directory + root_dir_id := fs_factory.fs_dir.set(root_dir)! + println('Created root directory with ID: ${root_dir_id}') + + // Update the filesystem with the root directory ID + my_fs.root_dir_id = root_dir_id + fs_factory.fs.set(my_fs)! + + // Create test directories with various parameters + mut test_dir1 := fs_factory.fs_dir.new( + name: 'test_dir1' + fs_id: fs_id + parent_id: root_dir_id + description: 'First test directory' + )! + + mut test_dir2 := fs_factory.fs_dir.new( + name: 'test_dir2' + fs_id: fs_id + parent_id: root_dir_id + description: 'Second test directory with tags' + tags: ['test', 'directory', 'example'] + )! + + mut test_dir3 := fs_factory.fs_dir.new( + name: 'test_dir3' + fs_id: fs_id + parent_id: root_dir_id + description: 'Third test directory with comments' + comments: [db.CommentArg{ + text: 'This is a test comment' + author: 'test_user' + }] + )! + + // Save the test directories + dir1_id := fs_factory.fs_dir.set(test_dir1)! + dir2_id := fs_factory.fs_dir.set(test_dir2)! + dir3_id := fs_factory.fs_dir.set(test_dir3)! + + println('Created test directories:') + println('- ${test_dir1.name} with ID: ${dir1_id}') + println('- ${test_dir2.name} with ID: ${dir2_id}') + println('- ${test_dir3.name} with ID: ${dir3_id}') + + // Test loading directories by ID + println('\nTesting directory loading...') + + loaded_root_dir := fs_factory.fs_dir.get(root_dir_id)! + println('Loaded root directory: ${loaded_root_dir.name} (ID: ${loaded_root_dir.id})') + + loaded_dir1 := fs_factory.fs_dir.get(dir1_id)! + println('Loaded test_dir1: ${loaded_dir1.name} (ID: ${loaded_dir1.id})') + println(' Description: ${loaded_dir1.description}') + + loaded_dir2 := fs_factory.fs_dir.get(dir2_id)! + println('Loaded test_dir2: ${loaded_dir2.name} (ID: ${loaded_dir2.id})') + println(' Description: ${loaded_dir2.description}') + println(' Tags: ${loaded_dir2.tags}') + + loaded_dir3 := fs_factory.fs_dir.get(dir3_id)! + println('Loaded test_dir3: ${loaded_dir3.name} (ID: ${loaded_dir3.id})') + println(' Description: ${loaded_dir3.description}') + + // Verify that loaded directories match the original ones + println('\nVerifying data integrity...') + + if loaded_root_dir.name == root_dir.name && loaded_root_dir.description == root_dir.description { + println('✓ Root directory data integrity verified') + } else { + println('✗ Root directory data integrity check failed') + } + + if loaded_dir1.name == test_dir1.name && loaded_dir1.description == test_dir1.description { + println('✓ Test directory 1 data integrity verified') + } else { + println('✗ Test directory 1 data integrity check failed') + } + + if loaded_dir2.name == test_dir2.name && loaded_dir2.description == test_dir2.description && loaded_dir2.tags == test_dir2.tags { + println('✓ Test directory 2 data integrity verified') + } else { + println('✗ Test directory 2 data integrity check failed') + } + + if loaded_dir3.name == test_dir3.name && loaded_dir3.description == test_dir3.description { + println('✓ Test directory 3 data integrity verified') + } else { + println('✗ Test directory 3 data integrity check failed') + } + + // Test exist method + println('\nTesting directory existence checks...') + + exists := fs_factory.fs_dir.exist(root_dir_id)! + println('Root directory exists: ${exists}') + + exists = fs_factory.fs_dir.exist(dir1_id)! + println('Test directory 1 exists: ${exists}') + + exists = fs_factory.fs_dir.exist(dir2_id)! + println('Test directory 2 exists: ${exists}') + + exists = fs_factory.fs_dir.exist(dir3_id)! + println('Test directory 3 exists: ${exists}') + + // Test with non-existent ID + exists = fs_factory.fs_dir.exist(999999)! + println('Non-existent directory exists: ${exists}') + + println('\nFsDir test completed successfully!') +} \ No newline at end of file diff --git a/lib/hero/herofs/factory.v b/lib/hero/herofs/factory.v index fbe4b3c5..ace4db27 100644 --- a/lib/hero/herofs/factory.v +++ b/lib/hero/herofs/factory.v @@ -5,34 +5,41 @@ import freeflowuniverse.herolib.hero.db @[heap] pub struct FsFactory { pub mut: - fs DBFs - fs_blob DBFsBlob + fs DBFs + fs_blob DBFsBlob fs_blob_membership DBFsBlobMembership - fs_dir DBFsDir - fs_file DBFsFile - fs_symlink DBFsSymlink + fs_dir DBFsDir + fs_file DBFsFile + fs_symlink DBFsSymlink } pub fn new() !FsFactory { mut mydb := db.new()! - return FsFactory{ - fs: DBFs{ + mut f := FsFactory{ + fs: DBFs{ db: &mydb } - fs_blob: DBFsBlob{ + fs_blob: DBFsBlob{ db: &mydb } fs_blob_membership: DBFsBlobMembership{ db: &mydb } - fs_dir: DBFsDir{ + fs_dir: DBFsDir{ db: &mydb } - fs_file: DBFsFile{ + fs_file: DBFsFile{ db: &mydb } - fs_symlink: DBFsSymlink{ + fs_symlink: DBFsSymlink{ db: &mydb } } + f.fs.factory = &f + f.fs_blob.factory = &f + f.fs_blob_membership.factory = &f + f.fs_dir.factory = &f + f.fs_file.factory = &f + f.fs_symlink.factory = &f + return f } diff --git a/lib/hero/herofs/fs.v b/lib/hero/herofs/fs.v index c8f66891..7273aab2 100644 --- a/lib/hero/herofs/fs.v +++ b/lib/hero/herofs/fs.v @@ -23,7 +23,8 @@ pub mut: pub struct DBFs { pub mut: - db &db.DB @[skip; str: skip] + db &db.DB @[skip; str: skip] + factory &FsFactory = unsafe { nil } @[skip; str: skip] } pub fn (self Fs) type_name() string { diff --git a/lib/hero/herofs/fs_blob.v b/lib/hero/herofs/fs_blob.v index 7ce466b6..d279776c 100644 --- a/lib/hero/herofs/fs_blob.v +++ b/lib/hero/herofs/fs_blob.v @@ -1,11 +1,9 @@ module herofs -import time import crypto.blake3 import freeflowuniverse.herolib.data.encoder import freeflowuniverse.herolib.data.ourtime import freeflowuniverse.herolib.hero.db -import freeflowuniverse.herolib.core.texttools // FsBlob represents binary data up to 1MB @[heap] @@ -19,7 +17,8 @@ pub mut: pub struct DBFsBlob { pub mut: - db &db.DB @[skip; str: skip] + db &db.DB @[skip; str: skip] + factory &FsFactory = unsafe { nil } @[skip; str: skip] } pub fn (self FsBlob) type_name() string { @@ -41,7 +40,7 @@ fn (mut self DBFsBlob) load(mut o FsBlob, mut e encoder.Decoder) ! { @[params] pub struct FsBlobArg { pub mut: - data []u8 @[required] + data []u8 @[required] } pub fn (mut blob FsBlob) calculate_hash() { @@ -64,10 +63,6 @@ pub fn (mut self DBFsBlob) new(args FsBlobArg) !FsBlob { o.calculate_hash() // Set base fields - o.name = args.name - o.description = args.description - o.tags = self.db.tags_get(args.tags)! - o.comments = self.db.comments_get(args.comments)! o.updated_at = ourtime.now().unix() return o @@ -90,7 +85,6 @@ pub fn (mut self DBFsBlob) set(o FsBlob) !u32 { return id } - pub fn (mut self DBFsBlob) delete(id u32) ! { // Get the blob to retrieve its hash mut blob := self.get(id)! @@ -102,10 +96,25 @@ pub fn (mut self DBFsBlob) delete(id u32) ! { self.db.delete[FsBlob](id)! } +pub fn (mut self DBFsBlob) delete_multi(ids []u32) ! { + for id in ids { + self.delete(id)! + } +} + pub fn (mut self DBFsBlob) exist(id u32) !bool { return self.db.exists[FsBlob](id)! } +pub fn (mut self DBFsBlob) exist_multi(ids []u32) !bool { + for id in ids { + if !self.exist(id)! { + return false + } + } + return true +} + pub fn (mut self DBFsBlob) get(id u32) !FsBlob { mut o, data := self.db.get_data[FsBlob](id)! mut e_decoder := encoder.decoder_new(data) @@ -113,17 +122,24 @@ pub fn (mut self DBFsBlob) get(id u32) !FsBlob { return o } -pub fn (mut self DBFsBlob) get_by_hash(hash string) !FsBlob { - id_str := self.db.redis.hget('fsblob:hashes', hash)! - if id_str == '' { - return error('Blob with hash "${hash}" not found') +pub fn (mut self DBFsBlob) get_multi(id []u32) ![]FsBlob { + mut blobs := []FsBlob{} + for i in id { + blobs << self.get(i)! } - return self.get(id_str.u32())! + return blobs +} + +pub fn (mut self DBFsBlob) get_by_hash(hash string) !FsBlob { + if self.factory.fs_blob_membership.exist(hash)! { + o := self.factory.fs_blob_membership.get(hash) or { panic('bug') } + return self.get(o.blobid)! + } + return error('Blob with hash ${hash} not found') } pub fn (mut self DBFsBlob) exists_by_hash(hash string) !bool { - id_str := self.db.redis.hget('fsblob:hashes', hash)! - return id_str != '' + return self.factory.fs_blob_membership.exist(hash) } pub fn (blob FsBlob) verify_integrity() bool { @@ -131,16 +147,7 @@ pub fn (blob FsBlob) verify_integrity() bool { return hash.hex()[..48] == blob.hash } -// verify checks the integrity of a blob by its ID or hash -// Returns true if the blob's data matches its stored hash -pub fn (mut self DBFsBlob) verify(id_or_hash string) !bool { - // Try to parse as ID first - if id_or_hash.is_int() { - blob := self.get(id_or_hash.int().u32())! - return blob.verify_integrity() - } - - // Otherwise treat as hash - blob := self.get_by_hash(id_or_hash)! +pub fn (mut self DBFsBlob) verify(hash string) !bool { + blob := self.get_by_hash(hash)! return blob.verify_integrity() } diff --git a/lib/hero/herofs/fs_blob_membership.v b/lib/hero/herofs/fs_blob_membership.v index e30d73d9..e38e3e61 100644 --- a/lib/hero/herofs/fs_blob_membership.v +++ b/lib/hero/herofs/fs_blob_membership.v @@ -1,24 +1,21 @@ module herofs -import time -import crypto.blake3 import freeflowuniverse.herolib.data.encoder -import freeflowuniverse.herolib.data.ourtime import freeflowuniverse.herolib.hero.db - -//FsBlobMembership represents membership of a blob in one or more filesystems, the key is the hash of the blob +// FsBlobMembership represents membership of a blob in one or more filesystems, the key is the hash of the blob @[heap] pub struct FsBlobMembership { pub mut: - hash string // blake192 hash of content - fsid []u32 //list of fs ids where this blob is used - blobid u32 // id of the blob + hash string // blake192 hash of content + fsid []u32 // list of fs ids where this blob is used + blobid u32 // id of the blob } pub struct DBFsBlobMembership { pub mut: - db &db.DB @[skip; str: skip] + db &db.DB @[skip; str: skip] + factory &FsFactory = unsafe { nil } @[skip; str: skip] } pub fn (self FsBlobMembership) type_name() string { @@ -40,9 +37,9 @@ fn (mut self DBFsBlobMembership) load(mut o FsBlobMembership, mut e encoder.Deco @[params] pub struct FsBlobMembershipArg { pub mut: - hash string @[required] - fsid []u32 @[required] - blobid u32 @[required] + hash string @[required] + fsid []u32 @[required] + blobid u32 @[required] } // get new blob membership, not from the DB @@ -58,21 +55,21 @@ pub fn (mut self DBFsBlobMembership) new(args FsBlobMembershipArg) !FsBlobMember pub fn (mut self DBFsBlobMembership) set(o FsBlobMembership) !string { // Validate that the blob exists - blob_exists := self.db.fs_blob.exists(o.blobid)! + blob_exists := self.factory.fs_blob.exists(o.blobid)! if !blob_exists { return error('Blob with ID ${o.blobid} does not exist') } // Validate that all filesystems exist for fs_id in o.fsid { - fs_exists := self.db.fs_file.exists(fs_id)! + fs_exists := self.factory.fs_file.exists(fs_id)! if !fs_exists { return error('Filesystem with ID ${fs_id} does not exist') } } // Encode the object - mut e_encoder := encoder.encoder_new() + mut e_encoder := encoder.new() o.dump(mut e_encoder)! // Store using hash as key in the blob_membership hset @@ -85,8 +82,7 @@ pub fn (mut self DBFsBlobMembership) delete(hash string) ! { } pub fn (mut self DBFsBlobMembership) exist(hash string) !bool { - result := self.db.redis.hexists('fs_blob_membership', hash)! - return result == 1 + return self.db.redis.hexists('fs_blob_membership', hash)! } pub fn (mut self DBFsBlobMembership) get(hash string) !FsBlobMembership { @@ -97,11 +93,11 @@ pub fn (mut self DBFsBlobMembership) get(hash string) !FsBlobMembership { } // Decode hex data back to bytes - data := data.bytes() + data2 := data.bytes() // Create object and decode mut o := FsBlobMembership{} - mut e_decoder := encoder.decoder_new(data) + mut e_decoder := encoder.decoder_new(data2) self.load(mut o, mut e_decoder)! return o @@ -116,34 +112,31 @@ pub fn (mut self DBFsBlobMembership) add_filesystem(hash string, fs_id u32) !str } mut membership := self.get(hash)! - + // Check if filesystem is already in the list if fs_id !in membership.fsid { membership.fsid << fs_id } - + return self.set(membership)! } // Remove a filesystem from an existing blob membership pub fn (mut self DBFsBlobMembership) remove_filesystem(hash string, fs_id u32) !string { mut membership := self.get(hash)! - + // Remove filesystem from the list membership.fsid = membership.fsid.filter(it != fs_id) - + // If no filesystems left, delete the membership entirely if membership.fsid.len == 0 { self.delete(hash)! return hash } - + return self.set(membership)! } - - - // BlobList represents a simplified blob structure for listing purposes pub struct BlobList { pub mut: @@ -158,18 +151,18 @@ pub fn (mut self DBFsBlobMembership) list(prefix string) ![]FsBlobMembership { mut result := []FsBlobMembership{} mut cursor := 0 mut count := 0 - + for { if count >= 10000 { break } - + // Use hscan with MATCH pattern and COUNT to iterate through the hash - new_cursor, values := self.db.redis.hscan('fs_blob_membership', cursor, - match: '${prefix}*', + new_cursor, values := self.db.redis.hscan('fs_blob_membership', cursor, + match: '${prefix}*' count: 100 )! - + // Process the returned field-value pairs // hscan returns alternating field-value pairs, so we iterate by 2 mut i := 0 @@ -177,20 +170,20 @@ pub fn (mut self DBFsBlobMembership) list(prefix string) ![]FsBlobMembership { hash := values[i] // Skip the value (we don't need it since we'll get the object by hash) i += 2 - + if hash.starts_with(prefix) { result << self.get(hash)! count++ } } - + // If cursor is "0", we've completed the full iteration if new_cursor == '0' { break } - + cursor = new_cursor.int() } - + return result } diff --git a/lib/hero/herofs/fs_dir.v b/lib/hero/herofs/fs_dir.v index c17aa21f..cd9c762c 100644 --- a/lib/hero/herofs/fs_dir.v +++ b/lib/hero/herofs/fs_dir.v @@ -12,8 +12,8 @@ import freeflowuniverse.herolib.hero.db pub struct FsDir { db.Base pub mut: - fs_id u32 // Associated filesystem - parent_id u32 // Parent directory ID (0 for root) + fs_id u32 // Associated filesystem + parent_id u32 // Parent directory ID (0 for root) directories []u32 files []u32 symlinks []u32 @@ -21,7 +21,8 @@ pub mut: pub struct DBFsDir { pub mut: - db &db.DB @[skip; str: skip] + db &db.DB @[skip; str: skip] + factory &FsFactory = unsafe { nil } @[skip; str: skip] } pub fn (self FsDir) type_name() string { @@ -29,22 +30,21 @@ pub fn (self FsDir) type_name() string { } pub fn (self FsDir) dump(mut e encoder.Encoder) ! { - e.add_u32(self.fs_id) e.add_u32(self.parent_id) - + // Handle directories array e.add_u16(u16(self.directories.len)) for dir_id in self.directories { e.add_u32(dir_id) } - + // Handle files array e.add_u16(u16(self.files.len)) for file_id in self.files { e.add_u32(file_id) } - + // Handle symlinks array e.add_u16(u16(self.symlinks.len)) for symlink_id in self.symlinks { @@ -55,21 +55,21 @@ pub fn (self FsDir) dump(mut e encoder.Encoder) ! { fn (mut self DBFsDir) load(mut o FsDir, mut e encoder.Decoder) ! { o.fs_id = e.get_u32()! o.parent_id = e.get_u32()! - + // Load directories array directories_count := e.get_u16()! o.directories = []u32{cap: int(directories_count)} for _ in 0 .. directories_count { o.directories << e.get_u32()! } - + // Load files array files_count := e.get_u16()! o.files = []u32{cap: int(files_count)} for _ in 0 .. files_count { o.files << e.get_u32()! } - + // Load symlinks array symlinks_count := e.get_u16()! o.symlinks = []u32{cap: int(symlinks_count)} @@ -121,14 +121,14 @@ pub fn (mut self DBFsDir) set(o FsDir) !u32 { pub fn (mut self DBFsDir) delete(id u32) ! { // Get the directory info before deleting dir := self.get(id)! - + // If has parent, remove from parent's directories list if dir.parent_id > 0 { - mut parent_dir := self.db.directories.get(dir.parent_id) or { + mut parent_dir := self.factory.fs_dir.get(dir.parent_id) or { return error('Parent directory with ID ${dir.parent_id} does not exist') } parent_dir.directories = parent_dir.directories.filter(it != id) - self.db.directories.set(parent_dir)! + self.factory.fs_dir.set(parent_dir)! } // Delete the directory itself self.db.delete[FsDir](id)! diff --git a/lib/hero/herofs/fs_file.v b/lib/hero/herofs/fs_file.v index 427d74fb..b8acd673 100644 --- a/lib/hero/herofs/fs_file.v +++ b/lib/hero/herofs/fs_file.v @@ -18,14 +18,15 @@ pub mut: blobs []u32 // IDs of file content blobs size_bytes u64 mime_type MimeType // MIME type as enum (MOVED FROM FsBlob) - checksum string // e.g., SHA256 checksum of the file + checksum string // e.g., SHA256 checksum of the file accessed_at i64 metadata map[string]string // Custom metadata } pub struct DBFsFile { pub mut: - db &db.DB @[skip; str: skip] + db &db.DB @[skip; str: skip] + factory &FsFactory = unsafe { nil } @[skip; str: skip] } pub fn (self FsFile) type_name() string { diff --git a/lib/hero/herofs/fs_symlink.v b/lib/hero/herofs/fs_symlink.v index 593962ac..a8fb4834 100644 --- a/lib/hero/herofs/fs_symlink.v +++ b/lib/hero/herofs/fs_symlink.v @@ -26,7 +26,8 @@ pub enum SymlinkTargetType { pub struct DBFsSymlink { pub mut: - db &db.DB @[skip; str: skip] + db &db.DB @[skip; str: skip] + factory &FsFactory = unsafe { nil } @[skip; str: skip] } pub fn (self FsSymlink) type_name() string {