add benchmarking, more models and examples

examples/Cargo.toml

@@ -6,16 +6,16 @@ publish = false # This is a package of examples, not meant to be published
 
 [dependencies]
 # Local Rhailib crates
-# Allows 'use rhai_client::...'
 rhai_client = { path = "../src/client" }
-# Allows 'use worker_lib::...'
-worker_lib = { path = "../src/worker", package = "worker" }
+worker = { path = "../src/worker" }
 
-# External dependencies (versions aligned with other crates)
-rhai = { version = "1.18.0", features = ["sync", "decimal"] }
-tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] }
+# External dependencies
+rhai = "1.18.0"
+tokio = { version = "1", features = ["full"] }
 log = "0.4"
 env_logger = "0.10"
+serde_json = "1.0"
+chrono = "0.4"
 
 [[bin]]
 name = "example_math_worker"
@@ -24,3 +24,11 @@ path = "example_math_worker.rs"
 [[bin]]
 name = "example_string_worker"
 path = "example_string_worker.rs"
+
+[[bin]]
+name = "dedicated_reply_queue_demo"
+path = "dedicated_reply_queue_demo.rs"
+
+[[bin]]
+name = "lua_client_demo"
+path = "lua_client_demo.rs"

examples/dedicated_reply_queue_demo.rs

@@ -0,0 +1,113 @@
+use log::{info, error, debug};
+use rhai::Engine;
+use rhai_client::{RhaiClient, RhaiClientError}; // RhaiTaskDetails is not directly used
+use worker_lib::spawn_rhai_worker;
+use std::time::Duration;
+use tokio::sync::mpsc;
+use serde_json::Value;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+
+    let redis_url = "redis://127.0.0.1/";
+    let circle_name = "reply_demo_circle";
+    let script_to_run = "let x = 40; x + 2"; // Simple script
+
+    info!("Starting Dedicated Reply Queue Demo...");
+
+    // 1. Create a Rhai Engine for the worker
+    let engine = Engine::new();
+
+    // 2. Setup shutdown channel for the worker
+    let (shutdown_tx, shutdown_rx) = mpsc::channel::<()>(1);
+
+    // 3. Spawn the worker
+    info!("Spawning worker for circle: {}", circle_name);
+    let worker_handle = spawn_rhai_worker(
+        0, // circle_id (can be arbitrary for this demo)
+        circle_name.to_string(),
+        engine,
+        redis_url.to_string(),
+        shutdown_rx,
+        false, // preserve_tasks
+    );
+
+    // Give the worker a moment to start up and connect (optional, but good for demo)
+    tokio::time::sleep(Duration::from_millis(500)).await;
+
+    // 4. Create RhaiClient
+    info!("Creating RhaiClient...");
+    let client = match RhaiClient::new(redis_url) {
+        Ok(c) => c,
+        Err(e) => {
+            error!("Failed to create RhaiClient: {}", e);
+            // Attempt to shutdown worker before exiting
+            let _ = shutdown_tx.send(()).await;
+            let _ = worker_handle.await;
+            // Explicitly cast the error to the trait object to satisfy the return type
+            return Err(Box::new(e) as Box<dyn std::error::Error>);
+        }
+    };
+    info!("RhaiClient created.");
+
+    // 5. Submit script and await result using the new mechanism
+    let task_timeout = Duration::from_secs(10);
+    let client_rpc_id: Option<Value> = Some(serde_json::json!({ "demo_request_id": "reply_queue_test_001" }));
+
+    info!("Submitting script to circle '{}' and awaiting result...", circle_name);
+    info!("Script: {}", script_to_run);
+
+    match client
+        .submit_script_and_await_result(
+            circle_name,
+            script_to_run.to_string(),
+            client_rpc_id,
+            task_timeout,
+            // poll_interval is no longer needed
+        )
+        .await
+    {
+        Ok(details) => {
+            info!("Task completed successfully!");
+            debug!("Full Task Details: {:#?}", details);
+            // The task_id is not part of the returned RhaiTaskDetails struct.
+            // We could modify the client to return (task_id, details) if needed,
+            // but for this demo, we'll just log the content of the returned details.
+            info!("Received details for script: {}", details.script);
+            info!("Status: {}", details.status);
+            if let Some(output) = details.output {
+                info!("Output: {}", output); // Expected: 42
+                assert_eq!(output, "42");
+            } else {
+                error!("Expected output, but got None.");
+            }
+            if let Some(error_msg) = details.error {
+                error!("Error: {}", error_msg);
+            }
+        }
+        Err(e) => {
+            error!("An error occurred while awaiting task result: {}", e);
+            // The specific error can be inspected if needed, e.g., for timeout
+            if let RhaiClientError::Timeout(task_id) = e {
+                info!("Task {} timed out.", task_id);
+            }
+        }
+    }
+
+    // 6. Shutdown the worker
+    info!("Sending shutdown signal to worker...");
+    if shutdown_tx.send(()).await.is_err() {
+        error!("Failed to send shutdown signal to worker. It might have already exited.");
+    }
+
+    info!("Waiting for worker to complete...");
+    match worker_handle.await {
+        Ok(Ok(_)) => info!("Worker exited successfully."),
+        Ok(Err(e)) => error!("Worker exited with an error: {}", e),
+        Err(e) => error!("Worker task panicked or was cancelled: {}", e),
+    }
+
+    info!("Dedicated Reply Queue Demo finished.");
+    Ok(())
+}

examples/example_math_worker.rs

@@ -1,8 +1,9 @@
 use rhai::Engine;
 use rhai_client::RhaiClient; // To submit tasks
-use worker_lib::{run_worker_loop, Args as WorkerArgs}; // To run the worker
+
 use std::time::Duration;
 use tokio::time::sleep;
+use worker_lib::spawn_rhai_worker;
 
 // Custom function for Rhai
 fn add(a: i64, b: i64) -> i64 {
@@ -19,17 +20,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     math_engine.register_fn("add", add);
     log::info!("Custom 'add' function registered with Rhai engine for Math Worker.");
 
-    let worker_args = WorkerArgs {
-        redis_url: "redis://127.0.0.1/".to_string(),
-        circles: vec!["math_circle".to_string()], // Worker listens on a specific queue
-    };
-    let worker_args_clone = worker_args.clone(); // Clone for the worker task
-
+    let (shutdown_tx, shutdown_rx) = tokio::sync::mpsc::channel(1);
     tokio::spawn(async move {
         log::info!("Math Worker task starting...");
-        if let Err(e) = run_worker_loop(math_engine, worker_args_clone).await {
-            log::error!("Math Worker loop failed: {}", e);
-        }
+        let _worker_handle = spawn_rhai_worker(
+            0,
+            "math_circle".to_string(),
+            math_engine,
+            "redis://127.0.0.1/".to_string(),
+            shutdown_rx,
+            false,
+        );
     });
 
     // Give the worker a moment to start and connect
@@ -47,14 +48,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     log::info!("Submitting math script to 'math_circle' and awaiting result...");
     
     let timeout_duration = Duration::from_secs(10);
-    let poll_interval = Duration::from_millis(500);
+
 
     match client.submit_script_and_await_result(
         "math_circle",
         script_content.to_string(),
         None,
-        timeout_duration,
-        poll_interval
+        timeout_duration
     ).await {
         Ok(details) => {
             log::info!("Math Worker Example: Task finished. Status: {}, Output: {:?}, Error: {:?}",

examples/example_string_worker.rs

@@ -1,8 +1,9 @@
 use rhai::Engine;
 use rhai_client::RhaiClient; // To submit tasks
-use worker_lib::{run_worker_loop, Args as WorkerArgs}; // To run the worker
+
 use std::time::Duration;
 use tokio::time::sleep;
+use worker_lib::spawn_rhai_worker;
 
 // Custom function for Rhai
 fn reverse_string(s: String) -> String {
@@ -19,17 +20,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     string_engine.register_fn("reverse_it", reverse_string);
     log::info!("Custom 'reverse_it' function registered with Rhai engine for String Worker.");
 
-    let worker_args = WorkerArgs {
-        redis_url: "redis://127.0.0.1/".to_string(),
-        circles: vec!["string_circle".to_string()], // Worker listens on a specific queue
-    };
-    let worker_args_clone = worker_args.clone();
-
+    let (shutdown_tx, shutdown_rx) = tokio::sync::mpsc::channel(1);
     tokio::spawn(async move {
         log::info!("String Worker task starting...");
-        if let Err(e) = run_worker_loop(string_engine, worker_args_clone).await {
-            log::error!("String Worker loop failed: {}", e);
-        }
+        let _worker_handle = spawn_rhai_worker(
+            0,
+            "string_circle".to_string(),
+            string_engine,
+            "redis://127.0.0.1/".to_string(),
+            shutdown_rx,
+            false,
+        );
     });
 
     // Give the worker a moment to start and connect
@@ -47,20 +48,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     log::info!("Submitting string script to 'string_circle' and awaiting result...");
 
     let timeout_duration = Duration::from_secs(10);
-    let poll_interval = Duration::from_millis(500);
+
 
     match client.submit_script_and_await_result(
         "string_circle",
         script_content.to_string(),
         None,
-        timeout_duration,
-        poll_interval
+        timeout_duration
     ).await {
         Ok(details) => {
             log::info!("String Worker Example: Task finished. Status: {}, Output: {:?}, Error: {:?}",
                 details.status, details.output, details.error);
             if details.status == "completed" {
-                assert_eq!(details.output, Some("\"dlrow olleh\"".to_string())); // Rhai strings include quotes in `debug` format
+                assert_eq!(details.output, Some("dlrow olleh".to_string()));
                 log::info!("String Worker Example: Assertion for output \"dlrow olleh\" passed!");
                 Ok(())
             } else {

examples/lua_client_demo.rs

@@ -0,0 +1,52 @@
+use worker_lib::spawn_rhai_worker;
+use rhai::Engine;
+use tokio::sync::mpsc;
+use tokio::signal;
+use log::info;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Initialize the logger
+    env_logger::init();
+
+    let redis_url = "redis://127.0.0.1/";
+    let circle_name = "default".to_string();
+    let mut engine = Engine::new(); // Create a new, simple Rhai engine
+
+    // Register a simple 'ping' function for the readiness check.
+    engine.register_fn("ping", || -> String {
+        "pong".to_string()
+    });
+
+    // Create a channel for the shutdown signal
+    let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
+
+    info!("Spawning Rhai worker for circle: {}", circle_name);
+
+    // Spawn the worker
+    let worker_handle = spawn_rhai_worker(
+        1, // circle_id
+        circle_name.clone(),
+        engine,
+        redis_url.to_string(),
+        shutdown_rx,
+        false, // preserve_tasks
+    );
+
+    info!("Worker spawned. Press Ctrl+C to shut down.");
+
+    // Wait for Ctrl+C
+    signal::ctrl_c().await?;
+
+    info!("Ctrl+C received. Sending shutdown signal to worker.");
+    let _ = shutdown_tx.send(()).await;
+
+    // Wait for the worker to finish
+    if let Err(e) = worker_handle.await? {
+        eprintln!("Worker process finished with an error: {:?}", e);
+    }
+
+    info!("Worker has shut down gracefully.");
+
+    Ok(())
+}

examples/run_benchmark_demo.rs

@@ -0,0 +1,133 @@
+//! Demo script showing how to run the hybrid performance benchmark
+//! 
+//! This example demonstrates:
+//! 1. Starting workers programmatically
+//! 2. Running the Lua batch script
+//! 3. Collecting and displaying statistics
+
+use rhailib::{RedisStatsCollector, WorkerManager, clear_redis_test_data, check_redis_connection};
+use redis::{Client, Commands};
+use std::fs;
+use std::time::Duration;
+
+const REDIS_URL: &str = "redis://localhost:6379";
+const CIRCLE_NAME: &str = "demo_circle";
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    env_logger::init();
+    
+    println!("🚀 Rhailib Hybrid Performance Benchmark Demo");
+    println!("============================================");
+    
+    // Check Redis connection
+    println!("📡 Checking Redis connection...");
+    check_redis_connection(REDIS_URL)?;
+    println!("✅ Redis connection successful");
+    
+    // Clear any existing test data
+    println!("🧹 Clearing existing test data...");
+    clear_redis_test_data(REDIS_URL)?;
+    println!("✅ Test data cleared");
+    
+    // Load Lua script
+    println!("📜 Loading Lua batch script...");
+    let lua_script = fs::read_to_string("scripts/run_rhai_batch.lua")?;
+    println!("✅ Lua script loaded ({} bytes)", lua_script.len());
+    
+    // Start workers
+    println!("👷 Starting 2 worker processes...");
+    let mut worker_manager = WorkerManager::new();
+    worker_manager.start_workers(2, CIRCLE_NAME, REDIS_URL)?;
+    worker_manager.wait_for_workers_ready(Duration::from_secs(3))?;
+    println!("✅ Workers started and ready");
+    
+    // Connect to Redis
+    let redis_client = Client::open(REDIS_URL)?;
+    let mut conn = redis_client.get_connection()?;
+    
+    // Execute batch workload
+    println!("🎯 Submitting batch of 100 tasks...");
+    let batch_id = format!("demo_batch_{}", chrono::Utc::now().timestamp_millis());
+    let simple_script = "let x = 42; x * 2";
+    
+    let start_time = std::time::Instant::now();
+    
+    let result: redis::Value = redis::cmd("EVAL")
+        .arg(&lua_script)
+        .arg(0) // No keys
+        .arg(CIRCLE_NAME)
+        .arg(100) // task count
+        .arg(simple_script)
+        .arg(&batch_id)
+        .query(&mut conn)?;
+    
+    let submission_time = start_time.elapsed();
+    println!("✅ Batch submitted in {:?}", submission_time);
+    
+    // Parse result
+    if let redis::Value::Data(data) = result {
+        let response: serde_json::Value = serde_json::from_slice(&data)?;
+        println!("📊 Batch info: {}", serde_json::to_string_pretty(&response)?);
+    }
+    
+    // Wait for completion and collect statistics
+    println!("⏳ Waiting for batch completion...");
+    let stats_collector = RedisStatsCollector::new(REDIS_URL)?;
+    
+    let completed = stats_collector.wait_for_batch_completion(
+        &batch_id,
+        100,
+        Duration::from_secs(30),
+    )?;
+    
+    if !completed {
+        println!("⚠️  Batch did not complete within timeout");
+        return Ok(());
+    }
+    
+    println!("✅ Batch completed!");
+    
+    // Collect and display statistics
+    println!("📈 Collecting performance statistics...");
+    let timings = stats_collector.collect_batch_timings(&batch_id)?;
+    let stats = stats_collector.calculate_stats(&timings);
+    
+    println!("\n📊 PERFORMANCE RESULTS");
+    println!("======================");
+    println!("Total tasks:       {}", stats.total_tasks);
+    println!("Completed tasks:   {}", stats.completed_tasks);
+    println!("Failed tasks:      {}", stats.failed_tasks);
+    println!("Error rate:        {:.2}%", stats.error_rate);
+    println!("Throughput:        {:.2} tasks/second", stats.throughput_tps);
+    println!("Batch duration:    {:.2} ms", stats.batch_duration_ms);
+    println!("\nLatency Statistics:");
+    println!("  Min:             {:.2} ms", stats.latency_stats.min_ms);
+    println!("  Max:             {:.2} ms", stats.latency_stats.max_ms);
+    println!("  Mean:            {:.2} ms", stats.latency_stats.mean_ms);
+    println!("  Median:          {:.2} ms", stats.latency_stats.median_ms);
+    println!("  P95:             {:.2} ms", stats.latency_stats.p95_ms);
+    println!("  P99:             {:.2} ms", stats.latency_stats.p99_ms);
+    println!("  Std Dev:         {:.2} ms", stats.latency_stats.std_dev_ms);
+    
+    // Show some individual task timings
+    println!("\n🔍 Sample Task Timings (first 10):");
+    for (i, timing) in timings.iter().take(10).enumerate() {
+        println!("  Task {}: {} -> {} ({:.2}ms, status: {})", 
+            i + 1,
+            timing.task_id,
+            timing.status,
+            timing.latency_ms,
+            timing.status
+        );
+    }
+    
+    // Cleanup
+    println!("\n🧹 Cleaning up...");
+    stats_collector.cleanup_batch_data(&batch_id)?;
+    worker_manager.shutdown()?;
+    println!("✅ Cleanup complete");
+    
+    println!("\n🎉 Demo completed successfully!");
+    
+    Ok(())
+}

examples/run_lua_demo.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Exit immediately if a command exits with a non-zero status.
+set -e
+
+CIRCLE_NAME="default"
+TASK_QUEUE_KEY="rhai_tasks:${CIRCLE_NAME}"
+
+# --- 1. Clean Up Previous State ---
+echo "Cleaning up previous Redis state..."
+redis-cli DEL "$TASK_QUEUE_KEY" > /dev/null 2>&1 || true
+echo "Redis state cleaned."
+
+# --- 2. Compile and Run the Worker ---
+echo "Compiling and running the 'lua_client_demo' worker in the background..."
+export RUST_LOG=info
+cargo run -p rhailib-examples --bin lua_client_demo &
+WORKER_PID=$!
+echo "Worker started with PID: $WORKER_PID"
+
+# --- 3. Wait for the Worker to be Ready using Ping-Pong ---
+echo "Waiting for worker to be ready (ping-pong check)..."
+ATTEMPTS=0
+MAX_ATTEMPTS=15 # Wait for a maximum of 15 seconds
+while [ $ATTEMPTS -lt $MAX_ATTEMPTS ]; do
+  # Send a 'ping()' script and check for a 'pong' in the output.
+  # The timeout for the ping itself is short (2 seconds).
+  PING_RESULT=$(redis-cli EVAL "$(cat ../scripts/run_rhai.lua)" 0 "$CIRCLE_NAME" "ping()" 2 || true)
+  if echo "$PING_RESULT" | grep -q "pong"; then
+    echo "Worker is ready. Received pong."
+    break
+  fi
+  echo "Ping failed or timed out. Retrying..."
+  sleep 1
+  ATTEMPTS=$((ATTEMPTS + 1))
+done
+
+if [ $ATTEMPTS -eq $MAX_ATTEMPTS ]; then
+  echo "Error: Timed out waiting for worker to respond to ping."
+  kill $WORKER_PID
+  exit 1
+fi
+
+# --- 4. Execute the Actual Script ---
+echo ""
+echo "Executing main Rhai script..."
+RESULT=$(redis-cli EVAL "$(cat ../scripts/run_rhai.lua)" 0 "$CIRCLE_NAME" "let x = 100; x * 2" 5)
+echo "Result from main script: $RESULT"
+
+# --- 5. Shutdown the Worker ---
+echo ""
+echo "Shutting down the worker (PID: $WORKER_PID)..."
+kill $WORKER_PID
+wait $WORKER_PID
+echo "Worker shut down."
+
+echo ""
+if echo "$RESULT" | grep -q "error"; then
+  echo "Demo completed with an error."
+  exit 1
+else
+  echo "Demo completed successfully."
+fi