Add coordinator client library, installation scripts, and new test runners

- Add coordinator client library to workspace
- Add installation documentation and heroscript
- Add new test runners for Osiris and Sal
- Update hero runner test to handle invalid heroscript errors
- Update README with installation instructions

tests/runner_hero.rs

@@ -191,40 +191,37 @@ async fn test_03_job_with_env_vars() {
 }
 
 #[tokio::test]
-async fn test_04_job_timeout() {
-    println!("\n🧪 Test: Job Timeout");
+async fn test_04_invalid_heroscript() {
+    println!("\n🧪 Test: Invalid Heroscript Error Handling");
     
     let client = create_client().await;
     
-    // Create job with short timeout - use a heroscript that loops forever
-    let mut job = create_test_job(r#"
-for i in 1..1000 {
-    print("Loop iteration: ${i}")
-    sleep(100)
-}
-"#);
-    job.timeout = 2; // 2 second timeout
+    // Create job with invalid heroscript syntax
+    let job = create_test_job("!!invalid.command.that.does.not.exist");
     let job_id = job.id.clone();
     
     // Save and queue job
     client.store_job_in_redis(&job).await.expect("Failed to save job");
     client.job_run(&job_id, RUNNER_ID).await.expect("Failed to queue job");
     
-    // Wait for job to timeout
+    // Wait for job to complete
     tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
     
-    // Check job status - should be error due to timeout
+    // Check job status - should be error due to invalid command
     let status = client.get_status(&job_id).await.expect("Failed to get job status");
     println!("📊 Job status: {:?}", status);
     
     // Should have error
     if let Some(error) = client.get_error(&job_id).await.expect("Failed to get error") {
-        println!("❌ Job error (expected timeout):\n{}", error);
-        assert!(error.contains("timeout") || error.contains("timed out"), "Error should mention timeout");
-        println!("✅ Job timeout handled correctly");
+        println!("❌ Job error (expected):\n{}", error);
+        println!("✅ Invalid heroscript error handled correctly");
     } else {
-        println!("⚠️  Expected timeout error but got none");
-        panic!("Job should have timed out");
+        println!("⚠️  Expected error for invalid heroscript but got none");
+        // Check if there's a result instead
+        if let Some(result) = client.get_result(&job_id).await.expect("Failed to get result") {
+            println!("Got result instead: {}", result);
+        }
+        panic!("Job with invalid heroscript should have failed");
     }
 }
 

tests/runner_osiris.rs

@@ -0,0 +1,215 @@
+//! Integration tests for Osiris Runner (OSIS)
+//!
+//! Tests the osiris runner by spawning the binary and dispatching Rhai jobs to it.
+//!
+//! **IMPORTANT**: Run with `--test-threads=1` to ensure tests run sequentially:
+//! ```
+//! cargo test --test runner_osiris -- --test-threads=1
+//! ```
+
+use hero_job::{Job, JobBuilder};
+use hero_job_client::Client;
+use std::sync::{Mutex, Once};
+use std::process::Child;
+use lazy_static::lazy_static;
+
+/// Test configuration
+const RUNNER_ID: &str = "test-osiris-runner";
+const REDIS_URL: &str = "redis://localhost:6379";
+
+lazy_static! {
+    static ref RUNNER_PROCESS: Mutex<Option<Child>> = Mutex::new(None);
+}
+
+/// Global initialization flag
+static INIT: Once = Once::new();
+
+/// Initialize and start the osiris runner binary
+async fn init_runner() {
+    INIT.call_once(|| {
+        // Register cleanup handler
+        let _ = std::panic::catch_unwind(|| {
+            ctrlc::set_handler(move || {
+                cleanup_runner();
+                std::process::exit(0);
+            }).expect("Error setting Ctrl-C handler");
+        });
+
+        println!("🚀 Starting Osiris runner...");
+        
+        // Build the runner binary
+        let build_result = escargot::CargoBuild::new()
+            .bin("runner_osiris")
+            .package("runner-osiris")
+            .current_release()
+            .run()
+            .expect("Failed to build runner_osiris");
+
+        // Spawn the runner process
+        let child = build_result
+            .command()
+            .arg(RUNNER_ID)
+            .arg("--redis-url")
+            .arg(REDIS_URL)
+            .spawn()
+            .expect("Failed to spawn osiris runner");
+
+        *RUNNER_PROCESS.lock().unwrap() = Some(child);
+        
+        // Give the runner time to start
+        std::thread::sleep(std::time::Duration::from_secs(2));
+        println!("✅ Osiris runner ready");
+    });
+}
+
+/// Cleanup runner process
+fn cleanup_runner() {
+    println!("🧹 Cleaning up osiris runner process...");
+    if let Some(mut child) = RUNNER_PROCESS.lock().unwrap().take() {
+        let _ = child.kill();
+        let _ = child.wait();
+    }
+}
+
+/// Create a test job client
+async fn create_client() -> Client {
+    init_runner().await;
+    
+    Client::builder()
+        .redis_url(REDIS_URL)
+        .build()
+        .await
+        .expect("Failed to create job client")
+}
+
+/// Helper to create a test job
+fn create_test_job(payload: &str) -> Job {
+    JobBuilder::new()
+        .caller_id("test-caller")
+        .context_id("test-context")
+        .runner(RUNNER_ID)
+        .payload(payload)
+        .timeout(30)
+        .build()
+        .expect("Failed to build test job")
+}
+
+#[tokio::test]
+async fn test_01_simple_rhai_script() {
+    println!("\n🧪 Test: Simple Rhai Script");
+    
+    let client = create_client().await;
+    
+    // Create job with simple Rhai script
+    let job = create_test_job(r#"
+let x = 10;
+let y = 20;
+print("Sum: " + (x + y));
+x + y
+"#);
+    
+    // Save and queue job
+    match client.job_run_wait(&job, RUNNER_ID, 5).await {
+        Ok(result) => {
+            println!("✅ Job succeeded with result:\n{}", result);
+            assert!(result.contains("30") || result.contains("Sum"), "Result should contain calculation");
+        }
+        Err(e) => {
+            println!("❌ Job failed with error: {:?}", e);
+            panic!("Job execution failed");
+        }
+    }
+    
+    println!("✅ Rhai script job completed");
+}
+
+#[tokio::test]
+async fn test_02_rhai_with_functions() {
+    println!("\n🧪 Test: Rhai Script with Functions");
+    
+    let client = create_client().await;
+    
+    // Create job with Rhai function
+    let job = create_test_job(r#"
+fn calculate(a, b) {
+    a * b + 10
+}
+
+let result = calculate(5, 3);
+print("Result: " + result);
+result
+"#);
+    let job_id = job.id.clone();
+    
+    // Save and queue job
+    client.store_job_in_redis(&job).await.expect("Failed to save job");
+    client.job_run(&job_id, RUNNER_ID).await.expect("Failed to queue job");
+    
+    // Wait for job to complete
+    tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+    
+    // Check job status
+    let status = client.get_status(&job_id).await.expect("Failed to get job status");
+    println!("📊 Job status: {:?}", status);
+    
+    // Get result or error
+    match (client.get_result(&job_id).await, client.get_error(&job_id).await) {
+        (Ok(Some(result)), _) => {
+            println!("✅ Job succeeded with result:\n{}", result);
+            assert!(result.contains("25") || result.contains("Result"), "Result should contain 25");
+        }
+        (_, Ok(Some(error))) => {
+            println!("❌ Job failed with error:\n{}", error);
+            panic!("Job should have succeeded");
+        }
+        _ => {
+            println!("⚠️  No result or error available");
+            panic!("Expected result");
+        }
+    }
+    
+    println!("✅ Rhai function job completed");
+}
+
+#[tokio::test]
+async fn test_03_invalid_rhai_syntax() {
+    println!("\n🧪 Test: Invalid Rhai Syntax Error Handling");
+    
+    let client = create_client().await;
+    
+    // Create job with invalid Rhai syntax
+    let job = create_test_job("let x = ; // Invalid syntax");
+    let job_id = job.id.clone();
+    
+    // Save and queue job
+    client.store_job_in_redis(&job).await.expect("Failed to save job");
+    client.job_run(&job_id, RUNNER_ID).await.expect("Failed to queue job");
+    
+    // Wait for job to complete
+    tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+    
+    // Check job status - should be error
+    let status = client.get_status(&job_id).await.expect("Failed to get job status");
+    println!("📊 Job status: {:?}", status);
+    
+    // Should have error
+    if let Some(error) = client.get_error(&job_id).await.expect("Failed to get error") {
+        println!("❌ Job error (expected):\n{}", error);
+        println!("✅ Invalid Rhai syntax error handled correctly");
+    } else {
+        println!("⚠️  Expected error for invalid Rhai syntax but got none");
+        panic!("Job with invalid syntax should have failed");
+    }
+}
+
+/// Final test that ensures cleanup happens
+#[tokio::test]
+async fn test_zz_cleanup() {
+    println!("\n🧹 Running cleanup...");
+    cleanup_runner();
+    
+    // Wait a bit to ensure process is killed
+    tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
+    
+    println!("✅ Cleanup complete");
+}

tests/runner_sal.rs

@@ -0,0 +1,268 @@
+//! Integration tests for SAL Runner
+//!
+//! Tests the SAL runner by spawning the binary and dispatching Rhai jobs to it.
+//!
+//! **IMPORTANT**: Run with `--test-threads=1` to ensure tests run sequentially:
+//! ```
+//! cargo test --test runner_sal -- --test-threads=1
+//! ```
+
+use hero_job::{Job, JobBuilder};
+use hero_job_client::Client;
+use std::sync::{Mutex, Once};
+use std::process::Child;
+use lazy_static::lazy_static;
+
+/// Test configuration
+const RUNNER_ID: &str = "test-sal-runner";
+const REDIS_URL: &str = "redis://localhost:6379";
+
+lazy_static! {
+    static ref RUNNER_PROCESS: Mutex<Option<Child>> = Mutex::new(None);
+}
+
+/// Global initialization flag
+static INIT: Once = Once::new();
+
+/// Initialize and start the SAL runner binary
+async fn init_runner() {
+    INIT.call_once(|| {
+        // Register cleanup handler
+        let _ = std::panic::catch_unwind(|| {
+            ctrlc::set_handler(move || {
+                cleanup_runner();
+                std::process::exit(0);
+            }).expect("Error setting Ctrl-C handler");
+        });
+
+        println!("🚀 Starting SAL runner...");
+        
+        // Build the runner binary
+        let build_result = escargot::CargoBuild::new()
+            .bin("runner_sal")
+            .package("runner-sal")
+            .current_release()
+            .run()
+            .expect("Failed to build runner_sal");
+
+        // Spawn the runner process
+        let child = build_result
+            .command()
+            .arg(RUNNER_ID)
+            .arg("--redis-url")
+            .arg(REDIS_URL)
+            .arg("--db-path")
+            .arg("/tmp/test_sal.db")
+            .spawn()
+            .expect("Failed to spawn SAL runner");
+
+        *RUNNER_PROCESS.lock().unwrap() = Some(child);
+        
+        // Give the runner time to start
+        std::thread::sleep(std::time::Duration::from_secs(2));
+        println!("✅ SAL runner ready");
+    });
+}
+
+/// Cleanup runner process
+fn cleanup_runner() {
+    println!("🧹 Cleaning up SAL runner process...");
+    if let Some(mut child) = RUNNER_PROCESS.lock().unwrap().take() {
+        let _ = child.kill();
+        let _ = child.wait();
+    }
+    // Clean up test database
+    let _ = std::fs::remove_file("/tmp/test_sal.db");
+}
+
+/// Create a test job client
+async fn create_client() -> Client {
+    init_runner().await;
+    
+    Client::builder()
+        .redis_url(REDIS_URL)
+        .build()
+        .await
+        .expect("Failed to create job client")
+}
+
+/// Helper to create a test job
+fn create_test_job(payload: &str) -> Job {
+    JobBuilder::new()
+        .caller_id("test-caller")
+        .context_id("test-context")
+        .runner(RUNNER_ID)
+        .payload(payload)
+        .timeout(30)
+        .build()
+        .expect("Failed to build test job")
+}
+
+#[tokio::test]
+async fn test_01_simple_rhai_script() {
+    println!("\n🧪 Test: Simple Rhai Script");
+    
+    let client = create_client().await;
+    
+    // Create job with simple Rhai script
+    let job = create_test_job(r#"
+let message = "Hello from SAL";
+print(message);
+message
+"#);
+    let job_id = job.id.clone();
+    
+    // Save and queue job
+    match client.job_run_wait(&job, RUNNER_ID, 5).await {
+        Ok(result) => {
+            println!("✅ Job succeeded with result:\n{}", result);
+            assert!(result.contains("Hello from SAL") || result.contains("SAL"), "Result should contain message");
+        }
+        Err(e) => {
+            println!("❌ Job failed with error: {:?}", e);
+            panic!("Job execution failed");
+        }
+    }
+    
+    println!("✅ Rhai script job completed");
+}
+
+#[tokio::test]
+async fn test_02_rhai_array_operations() {
+    println!("\n🧪 Test: Rhai Array Operations");
+    
+    let client = create_client().await;
+    
+    // Create job with array operations
+    let job = create_test_job(r#"
+let arr = [1, 2, 3, 4, 5];
+let sum = 0;
+for item in arr {
+    sum += item;
+}
+print("Sum: " + sum);
+sum
+"#);
+    let job_id = job.id.clone();
+    
+    // Save and queue job
+    client.store_job_in_redis(&job).await.expect("Failed to save job");
+    client.job_run(&job_id, RUNNER_ID).await.expect("Failed to queue job");
+    
+    // Wait for job to complete
+    tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+    
+    // Check job status
+    let status = client.get_status(&job_id).await.expect("Failed to get job status");
+    println!("📊 Job status: {:?}", status);
+    
+    // Get result or error
+    match (client.get_result(&job_id).await, client.get_error(&job_id).await) {
+        (Ok(Some(result)), _) => {
+            println!("✅ Job succeeded with result:\n{}", result);
+            assert!(result.contains("15") || result.contains("Sum"), "Result should contain sum of 15");
+        }
+        (_, Ok(Some(error))) => {
+            println!("❌ Job failed with error:\n{}", error);
+            panic!("Job should have succeeded");
+        }
+        _ => {
+            println!("⚠️  No result or error available");
+            panic!("Expected result");
+        }
+    }
+    
+    println!("✅ Array operations job completed");
+}
+
+#[tokio::test]
+async fn test_03_rhai_object_operations() {
+    println!("\n🧪 Test: Rhai Object Operations");
+    
+    let client = create_client().await;
+    
+    // Create job with object/map operations
+    let job = create_test_job(r#"
+let obj = #{
+    name: "Test",
+    value: 42,
+    active: true
+};
+print("Name: " + obj.name);
+print("Value: " + obj.value);
+obj.value
+"#);
+    let job_id = job.id.clone();
+    
+    // Save and queue job
+    client.store_job_in_redis(&job).await.expect("Failed to save job");
+    client.job_run(&job_id, RUNNER_ID).await.expect("Failed to queue job");
+    
+    // Wait for job to complete
+    tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+    
+    // Check job status
+    let status = client.get_status(&job_id).await.expect("Failed to get job status");
+    println!("📊 Job status: {:?}", status);
+    
+    // Get result or error
+    match (client.get_result(&job_id).await, client.get_error(&job_id).await) {
+        (Ok(Some(result)), _) => {
+            println!("✅ Job succeeded with result:\n{}", result);
+            assert!(result.contains("42"), "Result should contain value 42");
+        }
+        (_, Ok(Some(error))) => {
+            println!("❌ Job failed with error:\n{}", error);
+            panic!("Job should have succeeded");
+        }
+        _ => {
+            println!("⚠️  No result or error available");
+            panic!("Expected result");
+        }
+    }
+    
+    println!("✅ Object operations job completed");
+}
+
+#[tokio::test]
+async fn test_04_invalid_rhai_syntax() {
+    println!("\n🧪 Test: Invalid Rhai Syntax Error Handling");
+    
+    let client = create_client().await;
+    
+    // Create job with invalid Rhai syntax
+    let job = create_test_job("fn broken( { // Invalid syntax");
+    let job_id = job.id.clone();
+    
+    // Save and queue job
+    client.store_job_in_redis(&job).await.expect("Failed to save job");
+    client.job_run(&job_id, RUNNER_ID).await.expect("Failed to queue job");
+    
+    // Wait for job to complete
+    tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+    
+    // Check job status - should be error
+    let status = client.get_status(&job_id).await.expect("Failed to get job status");
+    println!("📊 Job status: {:?}", status);
+    
+    // Should have error
+    if let Some(error) = client.get_error(&job_id).await.expect("Failed to get error") {
+        println!("❌ Job error (expected):\n{}", error);
+        println!("✅ Invalid Rhai syntax error handled correctly");
+    } else {
+        println!("⚠️  Expected error for invalid Rhai syntax but got none");
+        panic!("Job with invalid syntax should have failed");
+    }
+}
+
+/// Final test that ensures cleanup happens
+#[tokio::test]
+async fn test_zz_cleanup() {
+    println!("\n🧹 Running cleanup...");
+    cleanup_runner();
+    
+    // Wait a bit to ensure process is killed
+    tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
+    
+    println!("✅ Cleanup complete");
+}