feat: Implement MyceliumStreamer for distributed data synchronization

- Introduces `MyceliumStreamer` for synchronizing data across a
  Mycelium network, enabling distributed data access.
- Allows adding multiple worker nodes to the streamer for data
  replication and redundancy.
- Provides `write` and `read` methods for seamless data
  management across nodes.
This commit is contained in:
Mahmoud Emad
2025-03-02 20:25:29 +02:00
parent b40c366335
commit a23eb0ff0b
3 changed files with 187 additions and 80 deletions

View File

@@ -0,0 +1,80 @@
module ourdb
import freeflowuniverse.herolib.clients.mycelium
struct MyceliumStreamer {
pub mut:
master &OurDB @[skip; str: skip]
workers map[string]&OurDB @[skip; str: skip] // key is mycelium public key, value is ourdb
incremental_mode bool = true // default is true
}
pub struct NewStreamerArgs {
pub mut:
incremental_mode bool = true // default is true
}
fn new_db_streamer(args NewStreamerArgs) !OurDB {
return new(
record_nr_max: 16777216 - 1
record_size_max: 1024
path: '/tmp/ourdb1'
reset: true
incremental_mode: args.incremental_mode
)!
}
pub fn (mut s MyceliumStreamer) add_worker(public_key string) ! {
mut db := new_db_streamer(incremental_mode: s.incremental_mode)!
s.workers[public_key] = &db
}
pub fn new_streamer(args NewStreamerArgs) !MyceliumStreamer {
mut db := new_db_streamer(args)!
mut s := MyceliumStreamer{
master: &db
workers: {}
incremental_mode: args.incremental_mode
}
return s
}
@[params]
pub struct MyceliumRecordArgs {
pub:
id u32 @[required]
value string @[required]
}
pub fn (mut s MyceliumStreamer) write(record MyceliumRecordArgs) !u32 {
mut id := s.master.set(id: record.id, data: record.value.bytes()) or {
return error('Failed to set id ${record.id} to value ${record.value} due to: ${err}')
}
// Get updates from the beginning (id 0) to ensure complete sync
data := s.master.push_updates(0) or { return error('Failed to push updates due to: ${err}') }
// Broadcast to all workers
for _, mut worker in s.workers {
worker.sync_updates(data) or {
return error('Failed to sync updates to worker due to: ${err}')
}
}
return id
}
pub struct MyceliumReadArgs {
pub:
id u32 @[required]
worker_public_key string
}
pub fn (mut s MyceliumStreamer) read(args MyceliumReadArgs) ![]u8 {
if args.worker_public_key.len > 0 {
if mut worker := s.workers[args.worker_public_key] {
return worker.get(args.id)!
}
return error('Worker with public key ${args.worker_public_key} not found')
}
return s.master.get(args.id)!
}