feat: Implement MyceliumStreamer for distributed data synchronization
- Introduces `MyceliumStreamer` for synchronizing data across a Mycelium network, enabling distributed data access. - Allows adding multiple worker nodes to the streamer for data replication and redundancy. - Provides `write` and `read` methods for seamless data management across nodes.
This commit is contained in:
80
lib/data/ourdb/mycelium_streamer.v
Normal file
80
lib/data/ourdb/mycelium_streamer.v
Normal file
@@ -0,0 +1,80 @@
|
||||
module ourdb
|
||||
|
||||
import freeflowuniverse.herolib.clients.mycelium
|
||||
|
||||
struct MyceliumStreamer {
|
||||
pub mut:
|
||||
master &OurDB @[skip; str: skip]
|
||||
workers map[string]&OurDB @[skip; str: skip] // key is mycelium public key, value is ourdb
|
||||
incremental_mode bool = true // default is true
|
||||
}
|
||||
|
||||
pub struct NewStreamerArgs {
|
||||
pub mut:
|
||||
incremental_mode bool = true // default is true
|
||||
}
|
||||
|
||||
fn new_db_streamer(args NewStreamerArgs) !OurDB {
|
||||
return new(
|
||||
record_nr_max: 16777216 - 1
|
||||
record_size_max: 1024
|
||||
path: '/tmp/ourdb1'
|
||||
reset: true
|
||||
incremental_mode: args.incremental_mode
|
||||
)!
|
||||
}
|
||||
|
||||
pub fn (mut s MyceliumStreamer) add_worker(public_key string) ! {
|
||||
mut db := new_db_streamer(incremental_mode: s.incremental_mode)!
|
||||
s.workers[public_key] = &db
|
||||
}
|
||||
|
||||
pub fn new_streamer(args NewStreamerArgs) !MyceliumStreamer {
|
||||
mut db := new_db_streamer(args)!
|
||||
mut s := MyceliumStreamer{
|
||||
master: &db
|
||||
workers: {}
|
||||
incremental_mode: args.incremental_mode
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@[params]
|
||||
pub struct MyceliumRecordArgs {
|
||||
pub:
|
||||
id u32 @[required]
|
||||
value string @[required]
|
||||
}
|
||||
|
||||
pub fn (mut s MyceliumStreamer) write(record MyceliumRecordArgs) !u32 {
|
||||
mut id := s.master.set(id: record.id, data: record.value.bytes()) or {
|
||||
return error('Failed to set id ${record.id} to value ${record.value} due to: ${err}')
|
||||
}
|
||||
|
||||
// Get updates from the beginning (id 0) to ensure complete sync
|
||||
data := s.master.push_updates(0) or { return error('Failed to push updates due to: ${err}') }
|
||||
|
||||
// Broadcast to all workers
|
||||
for _, mut worker in s.workers {
|
||||
worker.sync_updates(data) or {
|
||||
return error('Failed to sync updates to worker due to: ${err}')
|
||||
}
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
pub struct MyceliumReadArgs {
|
||||
pub:
|
||||
id u32 @[required]
|
||||
worker_public_key string
|
||||
}
|
||||
|
||||
pub fn (mut s MyceliumStreamer) read(args MyceliumReadArgs) ![]u8 {
|
||||
if args.worker_public_key.len > 0 {
|
||||
if mut worker := s.workers[args.worker_public_key] {
|
||||
return worker.get(args.id)!
|
||||
}
|
||||
return error('Worker with public key ${args.worker_public_key} not found')
|
||||
}
|
||||
return s.master.get(args.id)!
|
||||
}
|
||||
Reference in New Issue
Block a user