add data packages and remove empty submodule
This commit is contained in:
Timur Gordon
51
packages/data/radixtree/examples/basic_usage.rs
Normal file
51
packages/data/radixtree/examples/basic_usage.rs
Normal file
@@ -0,0 +1,51 @@
|
||||
use radixtree::RadixTree;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() -> Result<(), radixtree::Error> {
|
||||
// Create a temporary directory for the database
|
||||
let db_path = std::env::temp_dir().join("radixtree_example");
|
||||
std::fs::create_dir_all(&db_path)?;
|
||||
|
||||
println!("Creating radix tree at: {}", db_path.display());
|
||||
|
||||
// Create a new radix tree
|
||||
let mut tree = RadixTree::new(db_path.to_str().unwrap(), true)?;
|
||||
|
||||
// Store some data
|
||||
println!("Storing data...");
|
||||
tree.set("hello", b"world".to_vec())?;
|
||||
tree.set("help", b"me".to_vec())?;
|
||||
tree.set("helicopter", b"flying".to_vec())?;
|
||||
|
||||
// Retrieve and print the data
|
||||
let value = tree.get("hello")?;
|
||||
println!("hello: {}", String::from_utf8_lossy(&value));
|
||||
|
||||
// Update a value
|
||||
println!("Updating value...");
|
||||
tree.update("hello", b"updated world".to_vec())?;
|
||||
|
||||
// Retrieve the updated value
|
||||
let updated_value = tree.get("hello")?;
|
||||
println!("hello (updated): {}", String::from_utf8_lossy(&updated_value));
|
||||
|
||||
// Delete a key
|
||||
println!("Deleting 'help'...");
|
||||
tree.delete("help")?;
|
||||
|
||||
// Try to retrieve the deleted key (should fail)
|
||||
match tree.get("help") {
|
||||
Ok(value) => println!("Unexpected: help still exists with value: {}", String::from_utf8_lossy(&value)),
|
||||
Err(e) => println!("As expected, help was deleted: {}", e),
|
||||
}
|
||||
|
||||
// Clean up (optional)
|
||||
if std::env::var("KEEP_DB").is_err() {
|
||||
std::fs::remove_dir_all(&db_path)?;
|
||||
println!("Cleaned up database directory");
|
||||
} else {
|
||||
println!("Database kept at: {}", db_path.display());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
121
packages/data/radixtree/examples/large_scale_test.rs
Normal file
121
packages/data/radixtree/examples/large_scale_test.rs
Normal file
@@ -0,0 +1,121 @@
|
||||
use radixtree::RadixTree;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::io::{self, Write};
|
||||
|
||||
// Use much smaller batches to avoid hitting OurDB's size limit
|
||||
const BATCH_SIZE: usize = 1_000;
|
||||
const NUM_BATCHES: usize = 1_000; // Total records: 1,000,000
|
||||
const PROGRESS_INTERVAL: usize = 100;
|
||||
|
||||
fn main() -> Result<(), radixtree::Error> {
|
||||
// Overall metrics
|
||||
let total_start_time = Instant::now();
|
||||
let mut total_records_inserted = 0;
|
||||
let mut batch_times = Vec::with_capacity(NUM_BATCHES);
|
||||
|
||||
println!("Will insert up to {} records in batches of {}",
|
||||
BATCH_SIZE * NUM_BATCHES, BATCH_SIZE);
|
||||
|
||||
// Process in batches to avoid OurDB size limits
|
||||
for batch in 0..NUM_BATCHES {
|
||||
// Create a new database for each batch
|
||||
let batch_path = std::env::temp_dir().join(format!("radixtree_batch_{}", batch));
|
||||
|
||||
// Clean up any existing database
|
||||
if batch_path.exists() {
|
||||
std::fs::remove_dir_all(&batch_path)?;
|
||||
}
|
||||
std::fs::create_dir_all(&batch_path)?;
|
||||
|
||||
println!("\nBatch {}/{}: Creating new radix tree...", batch + 1, NUM_BATCHES);
|
||||
let mut tree = RadixTree::new(batch_path.to_str().unwrap(), true)?;
|
||||
|
||||
let batch_start_time = Instant::now();
|
||||
let mut last_progress_time = Instant::now();
|
||||
let mut last_progress_count = 0;
|
||||
|
||||
// Insert records for this batch
|
||||
for i in 0..BATCH_SIZE {
|
||||
let global_index = batch * BATCH_SIZE + i;
|
||||
let key = format!("key:{:08}", global_index);
|
||||
let value = format!("val{}", global_index).into_bytes();
|
||||
|
||||
tree.set(&key, value)?;
|
||||
|
||||
// Show progress at intervals
|
||||
if (i + 1) % PROGRESS_INTERVAL == 0 || i == BATCH_SIZE - 1 {
|
||||
let records_since_last = i + 1 - last_progress_count;
|
||||
let time_since_last = last_progress_time.elapsed();
|
||||
let records_per_second = records_since_last as f64 / time_since_last.as_secs_f64();
|
||||
|
||||
print!("\rProgress: {}/{} records ({:.2}%) - {:.2} records/sec",
|
||||
i + 1, BATCH_SIZE,
|
||||
(i + 1) as f64 / BATCH_SIZE as f64 * 100.0,
|
||||
records_per_second);
|
||||
io::stdout().flush().unwrap();
|
||||
|
||||
last_progress_time = Instant::now();
|
||||
last_progress_count = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
let batch_duration = batch_start_time.elapsed();
|
||||
batch_times.push(batch_duration);
|
||||
total_records_inserted += BATCH_SIZE;
|
||||
|
||||
println!("\nBatch {}/{} completed in {:?} ({:.2} records/sec)",
|
||||
batch + 1, NUM_BATCHES,
|
||||
batch_duration,
|
||||
BATCH_SIZE as f64 / batch_duration.as_secs_f64());
|
||||
|
||||
// Test random access performance for this batch
|
||||
println!("Testing access performance for batch {}...", batch + 1);
|
||||
let mut total_get_time = Duration::new(0, 0);
|
||||
let num_samples = 100;
|
||||
|
||||
// Use a simple distribution pattern
|
||||
for i in 0..num_samples {
|
||||
// Distribute samples across the batch
|
||||
let sample_id = batch * BATCH_SIZE + (i * (BATCH_SIZE / num_samples));
|
||||
let key = format!("key:{:08}", sample_id);
|
||||
|
||||
let get_start = Instant::now();
|
||||
let _ = tree.get(&key)?;
|
||||
total_get_time += get_start.elapsed();
|
||||
}
|
||||
|
||||
println!("Average time to retrieve a record: {:?}",
|
||||
total_get_time / num_samples as u32);
|
||||
|
||||
// Test prefix search performance
|
||||
println!("Testing prefix search performance...");
|
||||
let prefix = format!("key:{:02}", batch % 100);
|
||||
|
||||
let list_start = Instant::now();
|
||||
let keys = tree.list(&prefix)?;
|
||||
let list_duration = list_start.elapsed();
|
||||
|
||||
println!("Found {} keys with prefix '{}' in {:?}",
|
||||
keys.len(), prefix, list_duration);
|
||||
}
|
||||
|
||||
// Overall performance summary
|
||||
let total_duration = total_start_time.elapsed();
|
||||
println!("\n\nPerformance Summary:");
|
||||
println!("Total time to insert {} records: {:?}", total_records_inserted, total_duration);
|
||||
println!("Average insertion rate: {:.2} records/second",
|
||||
total_records_inserted as f64 / total_duration.as_secs_f64());
|
||||
|
||||
// Show performance trend
|
||||
println!("\nPerformance Trend (batch number vs. time):");
|
||||
for (i, duration) in batch_times.iter().enumerate() {
|
||||
if i % 10 == 0 || i == batch_times.len() - 1 { // Only show every 10th point
|
||||
println!(" Batch {}: {:?} ({:.2} records/sec)",
|
||||
i + 1,
|
||||
duration,
|
||||
BATCH_SIZE as f64 / duration.as_secs_f64());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
134
packages/data/radixtree/examples/performance_test.rs
Normal file
134
packages/data/radixtree/examples/performance_test.rs
Normal file
@@ -0,0 +1,134 @@
|
||||
use radixtree::RadixTree;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::io::{self, Write};
|
||||
|
||||
// Number of records to insert
|
||||
const TOTAL_RECORDS: usize = 1_000_000;
|
||||
// How often to report progress (every X records)
|
||||
const PROGRESS_INTERVAL: usize = 10_000;
|
||||
// How many records to use for performance sampling
|
||||
const PERFORMANCE_SAMPLE_SIZE: usize = 1000;
|
||||
|
||||
fn main() -> Result<(), radixtree::Error> {
|
||||
// Create a temporary directory for the database
|
||||
let db_path = std::env::temp_dir().join("radixtree_performance_test");
|
||||
|
||||
// Completely remove and recreate the directory to ensure a clean start
|
||||
if db_path.exists() {
|
||||
std::fs::remove_dir_all(&db_path)?;
|
||||
}
|
||||
std::fs::create_dir_all(&db_path)?;
|
||||
|
||||
println!("Creating radix tree at: {}", db_path.display());
|
||||
println!("Will insert {} records and show progress...", TOTAL_RECORDS);
|
||||
|
||||
// Create a new radix tree
|
||||
let mut tree = RadixTree::new(db_path.to_str().unwrap(), true)?;
|
||||
|
||||
// Track overall time
|
||||
let start_time = Instant::now();
|
||||
|
||||
// Track performance metrics
|
||||
let mut insertion_times = Vec::with_capacity(TOTAL_RECORDS / PROGRESS_INTERVAL);
|
||||
let mut last_batch_time = Instant::now();
|
||||
let mut last_batch_records = 0;
|
||||
|
||||
// Insert records and track progress
|
||||
for i in 0..TOTAL_RECORDS {
|
||||
let key = format!("key:{:08}", i);
|
||||
// Use smaller values to avoid exceeding OurDB's size limit
|
||||
let value = format!("val{}", i).into_bytes();
|
||||
|
||||
// Time the insertion of every Nth record for performance sampling
|
||||
if i % PERFORMANCE_SAMPLE_SIZE == 0 {
|
||||
let insert_start = Instant::now();
|
||||
tree.set(&key, value)?;
|
||||
let insert_duration = insert_start.elapsed();
|
||||
|
||||
// Only print detailed timing for specific samples to avoid flooding output
|
||||
if i % (PERFORMANCE_SAMPLE_SIZE * 10) == 0 {
|
||||
println!("Record {}: Insertion took {:?}", i, insert_duration);
|
||||
}
|
||||
} else {
|
||||
tree.set(&key, value)?;
|
||||
}
|
||||
|
||||
// Show progress at intervals
|
||||
if (i + 1) % PROGRESS_INTERVAL == 0 || i == TOTAL_RECORDS - 1 {
|
||||
let records_in_batch = i + 1 - last_batch_records;
|
||||
let batch_duration = last_batch_time.elapsed();
|
||||
let records_per_second = records_in_batch as f64 / batch_duration.as_secs_f64();
|
||||
|
||||
insertion_times.push((i + 1, batch_duration));
|
||||
|
||||
print!("\rProgress: {}/{} records ({:.2}%) - {:.2} records/sec",
|
||||
i + 1, TOTAL_RECORDS,
|
||||
(i + 1) as f64 / TOTAL_RECORDS as f64 * 100.0,
|
||||
records_per_second);
|
||||
io::stdout().flush().unwrap();
|
||||
|
||||
last_batch_time = Instant::now();
|
||||
last_batch_records = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
let total_duration = start_time.elapsed();
|
||||
println!("\n\nPerformance Summary:");
|
||||
println!("Total time to insert {} records: {:?}", TOTAL_RECORDS, total_duration);
|
||||
println!("Average insertion rate: {:.2} records/second",
|
||||
TOTAL_RECORDS as f64 / total_duration.as_secs_f64());
|
||||
|
||||
// Show performance trend
|
||||
println!("\nPerformance Trend (records inserted vs. time per batch):");
|
||||
for (i, (record_count, duration)) in insertion_times.iter().enumerate() {
|
||||
if i % 10 == 0 || i == insertion_times.len() - 1 { // Only show every 10th point to avoid too much output
|
||||
println!(" After {} records: {:?} for {} records ({:.2} records/sec)",
|
||||
record_count,
|
||||
duration,
|
||||
PROGRESS_INTERVAL,
|
||||
PROGRESS_INTERVAL as f64 / duration.as_secs_f64());
|
||||
}
|
||||
}
|
||||
|
||||
// Test access performance with distributed samples
|
||||
println!("\nTesting access performance with distributed samples...");
|
||||
let mut total_get_time = Duration::new(0, 0);
|
||||
let num_samples = 1000;
|
||||
|
||||
// Use a simple distribution pattern instead of random
|
||||
for i in 0..num_samples {
|
||||
// Distribute samples across the entire range
|
||||
let sample_id = (i * (TOTAL_RECORDS / num_samples)) % TOTAL_RECORDS;
|
||||
let key = format!("key:{:08}", sample_id);
|
||||
|
||||
let get_start = Instant::now();
|
||||
let _ = tree.get(&key)?;
|
||||
total_get_time += get_start.elapsed();
|
||||
}
|
||||
|
||||
println!("Average time to retrieve a record: {:?}",
|
||||
total_get_time / num_samples as u32);
|
||||
|
||||
// Test prefix search performance
|
||||
println!("\nTesting prefix search performance...");
|
||||
let prefixes = ["key:0", "key:1", "key:5", "key:9"];
|
||||
|
||||
for prefix in &prefixes {
|
||||
let list_start = Instant::now();
|
||||
let keys = tree.list(prefix)?;
|
||||
let list_duration = list_start.elapsed();
|
||||
|
||||
println!("Found {} keys with prefix '{}' in {:?}",
|
||||
keys.len(), prefix, list_duration);
|
||||
}
|
||||
|
||||
// Clean up (optional)
|
||||
if std::env::var("KEEP_DB").is_err() {
|
||||
std::fs::remove_dir_all(&db_path)?;
|
||||
println!("\nCleaned up database directory");
|
||||
} else {
|
||||
println!("\nDatabase kept at: {}", db_path.display());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
97
packages/data/radixtree/examples/prefix_operations.rs
Normal file
97
packages/data/radixtree/examples/prefix_operations.rs
Normal file
@@ -0,0 +1,97 @@
|
||||
use radixtree::RadixTree;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() -> Result<(), radixtree::Error> {
|
||||
// Create a temporary directory for the database
|
||||
let db_path = std::env::temp_dir().join("radixtree_prefix_example");
|
||||
std::fs::create_dir_all(&db_path)?;
|
||||
|
||||
println!("Creating radix tree at: {}", db_path.display());
|
||||
|
||||
// Create a new radix tree
|
||||
let mut tree = RadixTree::new(db_path.to_str().unwrap(), true)?;
|
||||
|
||||
// Store data with common prefixes
|
||||
println!("Storing data with common prefixes...");
|
||||
|
||||
// User data
|
||||
tree.set("user:1:name", b"Alice".to_vec())?;
|
||||
tree.set("user:1:email", b"alice@example.com".to_vec())?;
|
||||
tree.set("user:2:name", b"Bob".to_vec())?;
|
||||
tree.set("user:2:email", b"bob@example.com".to_vec())?;
|
||||
|
||||
// Post data
|
||||
tree.set("post:1:title", b"First Post".to_vec())?;
|
||||
tree.set("post:1:content", b"Hello World!".to_vec())?;
|
||||
tree.set("post:2:title", b"Second Post".to_vec())?;
|
||||
tree.set("post:2:content", b"Another post content".to_vec())?;
|
||||
|
||||
// Demonstrate listing keys with a prefix
|
||||
println!("\nListing keys with prefix 'user:1:'");
|
||||
let user1_keys = tree.list("user:1:")?;
|
||||
for key in &user1_keys {
|
||||
println!(" Key: {}", key);
|
||||
}
|
||||
|
||||
println!("\nListing keys with prefix 'post:'");
|
||||
let post_keys = tree.list("post:")?;
|
||||
for key in &post_keys {
|
||||
println!(" Key: {}", key);
|
||||
}
|
||||
|
||||
// Demonstrate getting all values with a prefix
|
||||
println!("\nGetting all values with prefix 'user:1:'");
|
||||
let user1_values = tree.getall("user:1:")?;
|
||||
for (i, value) in user1_values.iter().enumerate() {
|
||||
println!(" Value {}: {}", i + 1, String::from_utf8_lossy(value));
|
||||
}
|
||||
|
||||
// Demonstrate finding all user names
|
||||
println!("\nFinding all user names (prefix 'user:*:name')");
|
||||
let mut user_names = Vec::new();
|
||||
let all_keys = tree.list("user:")?;
|
||||
for key in all_keys {
|
||||
if key.ends_with(":name") {
|
||||
if let Ok(value) = tree.get(&key) {
|
||||
user_names.push((key, String::from_utf8_lossy(&value).to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (key, name) in user_names {
|
||||
println!(" {}: {}", key, name);
|
||||
}
|
||||
|
||||
// Demonstrate updating values with a specific prefix
|
||||
println!("\nUpdating all post titles...");
|
||||
let post_title_keys = tree.list("post:")?.into_iter().filter(|k| k.ends_with(":title")).collect::<Vec<_>>();
|
||||
|
||||
for key in post_title_keys {
|
||||
let old_value = tree.get(&key)?;
|
||||
let old_title = String::from_utf8_lossy(&old_value);
|
||||
let new_title = format!("UPDATED: {}", old_title);
|
||||
|
||||
println!(" Updating '{}' to '{}'", old_title, new_title);
|
||||
tree.update(&key, new_title.as_bytes().to_vec())?;
|
||||
}
|
||||
|
||||
// Verify updates
|
||||
println!("\nVerifying updates:");
|
||||
let post_keys = tree.list("post:")?;
|
||||
for key in post_keys {
|
||||
if key.ends_with(":title") {
|
||||
let value = tree.get(&key)?;
|
||||
println!(" {}: {}", key, String::from_utf8_lossy(&value));
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up (optional)
|
||||
if std::env::var("KEEP_DB").is_err() {
|
||||
std::fs::remove_dir_all(&db_path)?;
|
||||
println!("\nCleaned up database directory");
|
||||
} else {
|
||||
println!("\nDatabase kept at: {}", db_path.display());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user