MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

fine tuned vfs_db to work with dedupe db

Browse Source
This commit is contained in:
timurgordon
2025-02-28 03:15:47 +03:00
parent 400ea6e80e
commit a690f98cc1
6 changed files with 135 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
lib/vfs/vfs_db/encoder.v
View File

@@ -103,8 +103,8 @@ pub fn decode_directory(data []u8) !Directory {
// File encoding/decoding
// encode encodes a File to binary format
pub fn (f File) encode() []u8 {
// encode encodes a File metadata to binary format (without the actual file data)
pub fn (f File) encode(data_db_id ?u32) []u8 {
mut e := encoder.new()
e.add_u8(1) // version byte
e.add_u8(u8(vfs.FileType.file)) // type byte
@@ -114,15 +114,21 @@ pub fn (f File) encode() []u8 {
// Encode parent_id
e.add_u32(f.parent_id)
if id := data_db_id {
// only encode data_db_id if it's given
// if file has no data, it also doesn't have corresponding id in data_db
e.add_u32(id)
}
// Encode file data
e.add_string(f.data)
// Note: We no longer encode file data here
// The data ID will be appended by the save_entry function
return e.data
}
// decode_file decodes a binary format back to File
pub fn decode_file(data []u8) !File {
// decode_file decodes a binary format back to File (without the actual file data)
// returns file without data and the key of data in data db
pub fn decode_file_metadata(data []u8) !(File, ?u32) {
mut d := encoder.decoder_new(data)
version := d.get_u8()!
if version != 1 {
@@ -140,14 +146,20 @@ pub fn decode_file(data []u8) !File {
// Decode parent_id
parent_id := d.get_u32()!
// Decode file data
data_content := d.get_string()!
return File{
decoded_file := File{
metadata: metadata
parent_id: parent_id
data: data_content
data: '' // Empty data, will be loaded separately
}
if d.data.len == 0 {
return decoded_file, none
// means there was no data_db id stored with file, so is empty file
}
// Decode data ID reference
// This will be used to fetch the actual data from db_data
data_id := d.get_u32()!
return decoded_file, data_id
}
// Symlink encoding/decoding

38
lib/vfs/vfs_db/factory.v
View File

@@ -8,6 +8,7 @@ import freeflowuniverse.herolib.core.pathlib
pub struct VFSParams {
pub:
data_dir string // Directory to store DatabaseVFS data
metadata_dir string // Directory to store metadata (defaults to data_dir if not specified)
incremental_mode bool // Whether to enable incremental mode
}
@@ -16,12 +17,39 @@ pub fn new(mut database Database, params VFSParams) !&DatabaseVFS {
pathlib.get_dir(path: params.data_dir, create: true) or {
return error('Failed to create data directory: ${err}')
}
// Use the same database for both data and metadata if only one is provided
mut fs := &DatabaseVFS{
root_id: 1
block_size: 1024 * 4
data_dir: params.data_dir
db_data: database
root_id: 1
block_size: 1024 * 4
data_dir: params.data_dir
metadata_dir: if params.metadata_dir.len > 0 { params.metadata_dir } else{ params.data_dir}
db_data: database
db_metadata: database
}
return fs
}
// Factory method for creating a new DatabaseVFS instance with separate databases for data and metadata
pub fn new_with_separate_dbs(mut data_db Database, mut metadata_db Database, params VFSParams) !&DatabaseVFS {
pathlib.get_dir(path: params.data_dir, create: true) or {
return error('Failed to create data directory: ${err}')
}
if params.metadata_dir.len > 0 {
pathlib.get_dir(path: params.metadata_dir, create: true) or {
return error('Failed to create metadata directory: ${err}')
}
}
mut fs := &DatabaseVFS{
root_id: 1
block_size: 1024 * 4
data_dir: params.data_dir
metadata_dir: if params.metadata_dir.len > 0 { params.metadata_dir } else { params.data_dir}
db_data: data_db
db_metadata: metadata_db
}
return fs

19
lib/vfs/vfs_db/id_table.v Normal file
View File

@@ -0,0 +1,19 @@
module vfs_db
import freeflowuniverse.herolib.vfs
import freeflowuniverse.herolib.data.ourdb
import time
// get_database_id get's the corresponding db id for a file's metadata id.
// since multiple vfs can use single db, or db's can have their own id logic
// databases set independent id's to data
pub fn (fs DatabaseVFS) get_database_id(vfs_id u32) !u32 {
return fs.id_table[vfs_id] or { error('VFS ID ${vfs_id} not found.') }
}
// get_database_id get's the corresponding db id for a file's metadata id.
// since multiple vfs can use single db, or db's can have their own id logic
// databases set independent id's to data
pub fn (mut fs DatabaseVFS) set_database_id(vfs_id u32, db_id u32) ! {
fs.id_table[vfs_id] = db_id
}

57
lib/vfs/vfs_db/vfs.v
View File

@@ -2,6 +2,7 @@ module vfs_db
import freeflowuniverse.herolib.vfs
import freeflowuniverse.herolib.data.ourdb
import freeflowuniverse.herolib.data.encoder
import time
// DatabaseVFS represents the virtual filesystem
@@ -12,8 +13,10 @@ pub mut:
block_size u32 // Size of data blocks in bytes
data_dir string // Directory to store DatabaseVFS data
metadata_dir string // Directory where we store the metadata
db_data &Database @[str: skip] // Database instance for storage
db_data &Database @[str: skip] // Database instance for file data storage
db_metadata &Database @[str: skip] // Database instance for metadata storage
last_inserted_id u32
id_table map[u32]u32
}
pub interface Database {
@@ -30,25 +33,32 @@ pub fn (mut fs DatabaseVFS) get_next_id() u32 {
}
// load_entry loads an entry from the database by ID and sets up parent references
pub fn (mut fs DatabaseVFS) load_entry(id u32) !FSEntry {
if data := fs.db_data.get(id) {
pub fn (mut fs DatabaseVFS) load_entry(vfs_id u32) !FSEntry {
if metadata := fs.db_metadata.get(fs.get_database_id(vfs_id)!) {
// First byte is version, second byte indicates the type
// TODO: check we dont overflow filetype (u8 in boundaries of filetype)
entry_type := unsafe { vfs.FileType(data[1]) }
entry_type := unsafe { vfs.FileType(metadata[1]) }
match entry_type {
.directory {
mut dir := decode_directory(data) or {
mut dir := decode_directory(metadata) or {
return error('Failed to decode directory: ${err}')
}
return dir
}
.file {
mut file := decode_file(data) or { return error('Failed to decode file: ${err}') }
mut file, data_id := decode_file_metadata(metadata) or { return error('Failed to decode file: ${err}') }
if id := data_id {
// there was a data_db index stored with file so file has data
if file_data := fs.db_data.get(id) {
file.data = file_data.bytestr()
} else {
return error('This should never happen, data is not where its supposed to be')
}
}
return file
}
.symlink {
mut symlink := decode_symlink(data) or {
mut symlink := decode_symlink(metadata) or {
return error('Failed to decode symlink: ${err}')
}
return symlink
@@ -63,21 +73,42 @@ pub fn (mut fs DatabaseVFS) save_entry(entry FSEntry) !u32 {
match entry {
Directory {
encoded := entry.encode()
return fs.db_data.set(id: entry.metadata.id, data: encoded) or {
db_id := fs.db_metadata.set(id: entry.metadata.id, data: encoded) or {
return error('Failed to save directory on id:${entry.metadata.id}: ${err}')
}
fs.set_database_id(entry.metadata.id, db_id)!
return entry.metadata.id
}
File {
encoded := entry.encode()
return fs.db_data.set(id: entry.metadata.id, data: encoded) or {
return error('Failed to save file on id:${entry.metadata.id}: ${err}')
// First encode file data and store in db_data
data_encoded := entry.data.bytes()
metadata_bytes := if data_encoded.len == 0 {
entry.encode(none)
} else {
// file has data so that will be stored in data_db
// its corresponding id stored with file metadata
data_db_id := fs.db_data.set(id: entry.metadata.id, data: data_encoded) or {
return error('Failed to save file data on id:${entry.metadata.id}: ${err}')
}
// Encode the db_data ID in with the file metadata
entry.encode(data_db_id)
}
// Save the metadata_bytes to metadata_db
metadata_db_id := fs.db_metadata.set(id: entry.metadata.id, data: metadata_bytes) or {
return error('Failed to save file metadata on id:${entry.metadata.id}: ${err}')
}
fs.set_database_id(entry.metadata.id, metadata_db_id)!
return entry.metadata.id
}
Symlink {
encoded := entry.encode()
return fs.db_data.set(id: entry.metadata.id, data: encoded) or {
db_id := fs.db_metadata.set(id: entry.metadata.id, data: encoded) or {
return error('Failed to save symlink on id:${entry.metadata.id}: ${err}')
}
fs.set_database_id(entry.metadata.id, db_id)!
return entry.metadata.id
}
}
}

13
lib/vfs/vfs_db/vfs_directory.v
View File

@@ -143,7 +143,6 @@ pub fn (mut fs DatabaseVFS) directory_touch(dir_ Directory, name string) !&File
}
}
}
new_file := fs.new_file(
parent_id: dir.metadata.id
name: name
@@ -181,8 +180,16 @@ pub fn (mut fs DatabaseVFS) directory_rm(mut dir Directory, name string) ! {
return error('${name} not found')
}
// Delete entry from DB
fs.db_data.delete(found_id) or { return error('Failed to delete entry: ${err}') }
// get entry from db_metadata
metadata_bytes := fs.db_metadata.get(fs.get_database_id(found_id)!) or { return error('Failed to delete entry: ${err}') }
file, data_id := decode_file_metadata(metadata_bytes)!
if id := data_id {
// means file has associated data in db_data
fs.db_data.delete(id)!
}
fs.db_metadata.delete(file.metadata.id) or { return error('Failed to delete entry: ${err}') }
// Update children list
dir.children.delete(found_idx)

11
lib/vfs/vfs_db/vfs_getters.v
View File

@@ -7,11 +7,13 @@ import time
// Implementation of VFSImplementation interface
pub fn (mut fs DatabaseVFS) root_get_as_dir() !&Directory {
// Try to load root directory from DB if it exists
if data := fs.db_data.get(fs.root_id) {
mut loaded_root := decode_directory(data) or {
return error('Failed to decode root directory: ${err}')
if fs.root_id in fs.id_table {
if data := fs.db_metadata.get(fs.get_database_id(fs.root_id)!) {
mut loaded_root := decode_directory(data) or {
return error('Failed to decode root directory: ${err}')
}
return &loaded_root
}
return &loaded_root
}
// Create and save new root directory
@@ -44,7 +46,6 @@ fn (mut self DatabaseVFS) get_entry(path string) !FSEntry {
for i := 0; i < parts.len; i++ {
mut found := false
children := self.directory_children(mut current, false)!
for child in children {
if child.metadata.name == parts[i] {
match child {