WIP1: implementing JSON-RPC calls over Unix Sockets

2025-10-21 16:37:11 +02:00
parent c4ae52b6ff
commit 219e612eca
8 changed files with 265 additions and 40 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -901,7 +901,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d"
 dependencies = [
 "fastrand",
- "gloo-timers",
+ "gloo-timers 0.3.0",
 "tokio",
 ]

@@ -2310,6 +2310,12 @@ dependencies = [
 "const-random",
 ]

+[[package]]
+name = "doctest-file"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562"
+
 [[package]]
 name = "downcast-rs"
 version = "2.0.2"
@@ -2691,6 +2697,10 @@ name = "futures-timer"
 version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
+dependencies = [
+ "gloo-timers 0.2.6",
+ "send_wrapper",
+]

 [[package]]
 name = "futures-util"
@@ -2782,6 +2792,39 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"

+[[package]]
+name = "gloo-net"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c06f627b1a58ca3d42b45d6104bf1e1a03799df472df00988b6ba21accc10580"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-sink",
+ "gloo-utils",
+ "http 1.3.1",
+ "js-sys",
+ "pin-project",
+ "serde",
+ "serde_json",
+ "thiserror 1.0.69",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
+[[package]]
+name = "gloo-timers"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "gloo-timers"
 version = "0.3.0"
@@ -2794,6 +2837,19 @@ dependencies = [
 "wasm-bindgen",
 ]

+[[package]]
+name = "gloo-utils"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b5555354113b18c547c1d3a98fbf7fb32a9ff4f6fa112ce823a21641a0ba3aa"
+dependencies = [
+ "js-sys",
+ "serde",
+ "serde_json",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "h2"
 version = "0.3.27"
@@ -2912,6 +2968,7 @@ dependencies = [
 "rand 0.8.5",
 "redb",
 "redis",
+ "reth-ipc",
 "secrecy",
 "serde",
 "serde_json",
@@ -3400,6 +3457,21 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "interprocess"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d941b405bd2322993887859a8ee6ac9134945a24ec5ec763a8a962fc64dfec2d"
+dependencies = [
+ "doctest-file",
+ "futures-core",
+ "libc",
+ "recvmsg",
+ "tokio",
+ "widestring",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "intl-memoizer"
 version = "0.5.3"
@@ -3587,15 +3659,17 @@ dependencies = [

 [[package]]
 name = "jsonrpsee"
-version = "0.26.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f3f48dc3e6b8bd21e15436c1ddd0bc22a6a54e8ec46fedd6adf3425f396ec6a"
+checksum = "1fba77a59c4c644fd48732367624d1bcf6f409f9c9a286fbc71d2f1fc0b2ea16"
 dependencies = [
+ "jsonrpsee-client-transport",
 "jsonrpsee-core",
 "jsonrpsee-http-client",
 "jsonrpsee-proc-macros",
 "jsonrpsee-server",
 "jsonrpsee-types",
+ "jsonrpsee-wasm-client",
 "jsonrpsee-ws-client",
 "tokio",
 "tracing",
@@ -3603,12 +3677,14 @@ dependencies = [

 [[package]]
 name = "jsonrpsee-client-transport"
-version = "0.26.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf36eb27f8e13fa93dcb50ccb44c417e25b818cfa1a481b5470cd07b19c60b98"
+checksum = "a2a320a3f1464e4094f780c4d48413acd786ce5627aaaecfac9e9c7431d13ae1"
 dependencies = [
 "base64 0.22.1",
+ "futures-channel",
 "futures-util",
+ "gloo-net",
 "http 1.3.1",
 "jsonrpsee-core",
 "pin-project",
@@ -3626,9 +3702,9 @@ dependencies = [

 [[package]]
 name = "jsonrpsee-core"
-version = "0.26.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "316c96719901f05d1137f19ba598b5fe9c9bc39f4335f67f6be8613921946480"
+checksum = "693c93cbb7db25f4108ed121304b671a36002c2db67dff2ee4391a688c738547"
 dependencies = [
 "async-trait",
 "bytes",
@@ -3649,13 +3725,14 @@ dependencies = [
 "tokio-stream",
 "tower",
 "tracing",
+ "wasm-bindgen-futures",
 ]

 [[package]]
 name = "jsonrpsee-http-client"
-version = "0.26.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "790bedefcec85321e007ff3af84b4e417540d5c87b3c9779b9e247d1bcc3dab8"
+checksum = "6962d2bd295f75e97dd328891e58fce166894b974c1f7ce2e7597f02eeceb791"
 dependencies = [
 "base64 0.22.1",
 "http-body 1.0.1",
@@ -3676,9 +3753,9 @@ dependencies = [

 [[package]]
 name = "jsonrpsee-proc-macros"
-version = "0.26.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2da3f8ab5ce1bb124b6d082e62dffe997578ceaf0aeb9f3174a214589dc00f07"
+checksum = "2fa4f5daed39f982a1bb9d15449a28347490ad42b212f8eaa2a2a344a0dce9e9"
 dependencies = [
 "heck 0.5.0",
 "proc-macro-crate",
@@ -3689,9 +3766,9 @@ dependencies = [

 [[package]]
 name = "jsonrpsee-server"
-version = "0.26.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c51b7c290bb68ce3af2d029648148403863b982f138484a73f02a9dd52dbd7f"
+checksum = "d38b0bcf407ac68d241f90e2d46041e6a06988f97fe1721fb80b91c42584fae6"
 dependencies = [
 "futures-util",
 "http 1.3.1",
@@ -3716,9 +3793,9 @@ dependencies = [

 [[package]]
 name = "jsonrpsee-types"
-version = "0.26.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc88ff4688e43cc3fa9883a8a95c6fa27aa2e76c96e610b737b6554d650d7fd5"
+checksum = "66df7256371c45621b3b7d2fb23aea923d577616b9c0e9c0b950a6ea5c2be0ca"
 dependencies = [
 "http 1.3.1",
 "serde",
@@ -3727,10 +3804,22 @@ dependencies = [
 ]

 [[package]]
-name = "jsonrpsee-ws-client"
-version = "0.26.0"
+name = "jsonrpsee-wasm-client"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b6fceceeb05301cc4c065ab3bd2fa990d41ff4eb44e4ca1b30fa99c057c3e79"
+checksum = "6b67695cbcf4653f39f8f8738925547e0e23fd9fe315bccf951097b9f6a38781"
+dependencies = [
+ "jsonrpsee-client-transport",
+ "jsonrpsee-core",
+ "jsonrpsee-types",
+ "tower",
+]
+
+[[package]]
+name = "jsonrpsee-ws-client"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2da2694c9ff271a9d3ebfe520f6b36820e85133a51be77a3cb549fd615095261"
 dependencies = [
 "http 1.3.1",
 "jsonrpsee-client-transport",
@@ -5440,6 +5529,12 @@ dependencies = [
 "crossbeam-utils",
 ]

+[[package]]
+name = "recvmsg"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175"
+
 [[package]]
 name = "redb"
 version = "2.6.2"
@@ -5629,6 +5724,26 @@ dependencies = [
 "webpki-roots 1.0.2",
 ]

+[[package]]
+name = "reth-ipc"
+version = "1.6.0"
+source = "git+https://github.com/paradigmxyz/reth?rev=d8451e54e7267f9f1634118d6d279b2216f7e2bb#d8451e54e7267f9f1634118d6d279b2216f7e2bb"
+dependencies = [
+ "bytes",
+ "futures",
+ "futures-util",
+ "interprocess",
+ "jsonrpsee",
+ "pin-project",
+ "serde_json",
+ "thiserror 2.0.16",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower",
+ "tracing",
+]
+
 [[package]]
 name = "ring"
 version = "0.17.14"
@@ -6078,6 +6193,12 @@ version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"

+[[package]]
+name = "send_wrapper"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0"
+
 [[package]]
 name = "seq-macro"
 version = "0.3.6"
@@ -7481,6 +7602,12 @@ dependencies = [
 "rustls-pki-types",
 ]

+[[package]]
+name = "widestring"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471"
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,7 +25,7 @@ secrecy = "0.8"
 ed25519-dalek = "2"
 x25519-dalek = "2"
 base64 = "0.22"
-jsonrpsee = { version = "0.26.0", features = ["http-client", "ws-client", "server", "macros"] }
+jsonrpsee = { version = "0.25.1", features = ["http-client", "ws-client", "server", "macros"] }
 tantivy = "0.25.0"
 arrow-schema = "55.2.0"
 arrow-array = "55.2.0"
@@ -35,6 +35,7 @@ arrow = "55.2.0"
 lancedb = "0.22.1"
 uuid = "1.18.1"
 ureq = { version = "2.10.0", features = ["json", "tls"] }
+reth-ipc = { git = "https://github.com/paradigmxyz/reth", package = "reth-ipc", rev = "d8451e54e7267f9f1634118d6d279b2216f7e2bb" }

 [dev-dependencies]
 redis = { version = "0.24", features = ["aio", "tokio-comp"] }
--- a/README.md
+++ b/README.md
@@ -32,13 +32,31 @@ cargo build --release

 ### Running HeroDB

-Launch HeroDB with the required `--admin-secret` flag, which encrypts the admin database (DB 0) and authorizes admin access. Optional flags include `--dir` for the database directory, `--port` for the TCP port (default 6379), `--sled` for the sled backend, and `--enable-rpc` to start the JSON-RPC management server on port 8080.
+Launch HeroDB with the required `--admin-secret` flag, which encrypts the admin database (DB 0) and authorizes admin access. Optional flags include `--dir` for the database directory, `--port` for the TCP port (default 6379), `--sled` for the sled backend, `--enable-rpc` to start the HTTP JSON-RPC server on a TCP port, `--enable-rpc-ipc` to start JSON-RPC over a Unix Domain Socket (non-HTTP), and `--rpc-ipc-path <path>` to specify the socket path (default: `/tmp/herodb.ipc`).

 Example:
 ```bash
 ./target/release/herodb --dir /tmp/herodb --admin-secret myadminsecret --port 6379 --enable-rpc
 ```

+To enable JSON-RPC over a Unix Domain Socket at `/tmp/herodb.sock`:
+```bash
+./target/release/herodb --dir /tmp/herodb --admin-secret myadminsecret --enable-rpc-ipc --rpc-ipc-path /tmp/herodb.sock
+```
+
+Test the IPC endpoint interactively with socat:
+```bash
+sudo socat -d -d -t 5 - UNIX-CONNECT:/tmp/herodb.sock
+```
+Then paste a framed JSON-RPC request (Content-Length header, blank line, then JSON body). Example:
+```
+Content-Length: 73
+
+{"jsonrpc":"2.0","method":"hero_listDatabases","params":[],"id":3}
+```
+
+For a one-liner that auto-computes Content-Length and pretty-prints the JSON response, see docs/rpc_examples.md.
+
 For detailed launch options, see [Basics](docs/basics.md).

 ## Usage with Redis Clients
--- a/docs/basics.md
+++ b/docs/basics.md
@@ -9,8 +9,10 @@ To launch HeroDB, use the binary with required and optional flags. The `--admin-
 - `--port <port>`: TCP port for Redis protocol (default: 6379).
 - `--debug`: Enable debug logging.
 - `--sled`: Use Sled backend (default: Redb).
- `--enable-rpc`: Start JSON-RPC management server on port 8080.
+- `--enable-rpc`: Start JSON-RPC management server on port 8080 (HTTP over TCP).
 - `--rpc-port <port>`: Custom RPC port (default: 8080).
+- `--enable-rpc-ipc`: Start JSON-RPC over a Unix Domain Socket (non-HTTP).
+- `--rpc-ipc-path <path>`: Path to the Unix socket for IPC (default: `/tmp/herodb.ipc`).
 - `--admin-secret <secret>`: Required secret for DB 0 encryption and admin access.

 Example:
@@ -18,6 +20,23 @@ Example:
 ./target/release/herodb --dir /tmp/herodb --admin-secret mysecret --port 6379 --enable-rpc
 ```

+To enable JSON-RPC over a Unix Domain Socket at `/tmp/herodb.sock`:
+```bash
+./target/release/herodb --dir /tmp/herodb --admin-secret mysecret --enable-rpc-ipc --rpc-ipc-path /tmp/herodb.sock
+```
+
+Test the IPC endpoint interactively with socat (non-HTTP transport):
+```bash
+sudo socat -d -d -t 5 - UNIX-CONNECT:/tmp/herodb.sock
+```
+Then paste a framed JSON-RPC request (Content-Length header, blank line, then JSON body). Example:
+```
+Content-Length: 73
+
+{"jsonrpc":"2.0","method":"hero_listDatabases","params":[],"id":3}
+```
+More IPC examples are in [docs/rpc_examples.md](docs/rpc_examples.md).
+
 Deprecated flags (`--encrypt`, `--encryption-key`) are ignored for data DBs; per-database encryption is managed via RPC.

 ## Admin Database (DB 0)
--- a/docs/rpc_examples.md
+++ b/docs/rpc_examples.md
@@ -138,4 +138,30 @@ Returns stats like total databases and uptime.
 - Per-database encryption keys are write-only; set at creation and used transparently.
 - Access keys are hashed (SHA-256) for storage; provide plaintext in requests.
 - Backend options: `"Redb"` (default) or `"Sled"`.
- Config object fields (name, storage_path, etc.) are optional and currently ignored but positional.
+- Config object fields (name, storage_path, etc.) are optional and currently ignored but positional.
+
+## IPC over Unix Socket (non-HTTP)
+
+HeroDB supports JSON-RPC over a Unix Domain Socket using reth-ipc. This transport is not HTTP; messages are JSON-RPC framed with a Content-Length header.
+
+- Enable IPC on startup (adjust the socket path as needed):
+  - herodb --dir /path/to/data --admin-secret YOUR_SECRET --enable-rpc-ipc --rpc-ipc-path /tmp/herodb.sock
+
+- The same RPC methods are available as over HTTP. Namespace is "hero" (e.g. hero_listDatabases). See the RPC trait in [src/rpc.rs](src/rpc.rs) and CLI flags in [src/main.rs](src/main.rs). The IPC bootstrap is in [src/rpc_server.rs](src/rpc_server.rs).
+
+### Test via socat (interactive)
+
+1) Connect to the socket with a small timeout:
+```
+sudo socat -d -d -t 5 - UNIX-CONNECT:/tmp/herodb.sock
+```
+
+2) Paste a framed JSON-RPC request (Content-Length header, then a blank line, then the JSON body). For example to call hero_listDatabases:
+
+Content-Length: &lt;LEN-BYTES-OF-JSON&gt;
+
+{"jsonrpc":"2.0","id":3,"method":"hero_listDatabases","params":[]}
+
+Notes:
+- Replace &lt;LEN-BYTES-OF-JSON&gt; with the byte-length of the exact JSON payload you paste. There must be an empty line between the header and the JSON.
+- The response will appear in the same terminal, also framed with Content-Length.
--- a/src/main.rs
+++ b/src/main.rs
@@ -40,6 +40,14 @@ struct Args {
    #[arg(long, default_value = "8080")]
    rpc_port: u16,

+    /// Enable RPC over Unix Domain Socket (IPC)
+    #[arg(long)]
+    enable_rpc_ipc: bool,
+
+    /// RPC IPC socket path (Unix Domain Socket)
+    #[arg(long, default_value = "/tmp/herodb.ipc")]
+    rpc_ipc_path: String,
+
    /// Use the sled backend
    #[arg(long)]
    sled: bool,
@@ -105,7 +113,7 @@ async fn main() {
        let rpc_addr = format!("127.0.0.1:{}", args.rpc_port).parse().unwrap();
        let base_dir = args.dir.clone();

-        match rpc_server::start_rpc_server(rpc_addr, base_dir, backend, args.admin_secret.clone()).await {
+        match rpc_server::start_rpc_server(rpc_addr, base_dir, backend.clone(), args.admin_secret.clone()).await {
            Ok(handle) => {
                println!("RPC management server started on port {}", args.rpc_port);
                Some(handle)
@@ -119,6 +127,30 @@ async fn main() {
        None
    };

+    // Start IPC (Unix socket) RPC server if enabled
+    let _rpc_ipc_handle = if args.enable_rpc_ipc {
+        let base_dir = args.dir.clone();
+        let ipc_path = args.rpc_ipc_path.clone();
+
+        // Remove stale socket if present
+        if std::path::Path::new(&ipc_path).exists() {
+            let _ = std::fs::remove_file(&ipc_path);
+        }
+
+        match rpc_server::start_rpc_ipc_server(ipc_path.clone(), base_dir, backend.clone(), args.admin_secret.clone()).await {
+            Ok(handle) => {
+                println!("RPC IPC server started at {}", ipc_path);
+                Some(handle)
+            }
+            Err(e) => {
+                eprintln!("Failed to start RPC IPC server: {}", e);
+                None
+            }
+        }
+    } else {
+        None
+    };
+
    // accept new connections
    loop {
        let stream = listener.accept().await;
--- a/src/rpc.rs
+++ b/src/rpc.rs
@@ -71,7 +71,7 @@ pub fn hash_key(key: &str) -> String {
 }

 /// RPC trait for HeroDB management
-#[rpc(server, client, namespace = "hero")]
+#[rpc(server, client, namespace = "herodb")]
 pub trait Rpc {
    /// Create a new database with specified configuration
    #[method(name = "createDatabase")]
--- a/src/rpc_server.rs
+++ b/src/rpc_server.rs
@@ -2,6 +2,7 @@ use std::net::SocketAddr;
 use std::path::PathBuf;
 use jsonrpsee::server::{ServerBuilder, ServerHandle};
 use jsonrpsee::RpcModule;
+use reth_ipc::server::Builder as IpcServerBuilder;

 use crate::rpc::{RpcServer, RpcServerImpl};

@@ -27,24 +28,25 @@ pub async fn start_rpc_server(addr: SocketAddr, base_dir: PathBuf, backend: crat
    Ok(handle)
 }

-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::time::Duration;
+/// Start the JSON-RPC IPC server on the specified Unix socket endpoint
+pub async fn start_rpc_ipc_server(
+    endpoint: String,
+    base_dir: PathBuf,
+    backend: crate::options::BackendType,
+    admin_secret: String,
+) -> Result<ServerHandle, Box<dyn std::error::Error + Send + Sync>> {
+    // Create the RPC server implementation
+    let rpc_impl = RpcServerImpl::new(base_dir, backend, admin_secret);

-    #[tokio::test]
-    async fn test_rpc_server_startup() {
-        let addr = "127.0.0.1:0".parse().unwrap(); // Use port 0 for auto-assignment
-        let base_dir = PathBuf::from("/tmp/test_rpc");
-        let backend = crate::options::BackendType::Redb; // Default for test
+    // Create the RPC module
+    let mut module = RpcModule::new(());
+    module.merge(RpcServer::into_rpc(rpc_impl))?;

-        let handle = start_rpc_server(addr, base_dir, backend, "test-admin".to_string()).await.unwrap();
+    // Build the IPC server and start it
+    let server = IpcServerBuilder::default().build(endpoint.clone());
+    let handle = server.start(module).await?;

-        // Give the server a moment to start
-        tokio::time::sleep(Duration::from_millis(100)).await;
+    println!("RPC IPC server started on {}", endpoint);

-        // Stop the server
-        handle.stop().unwrap();
-        handle.stopped().await;
-    }
+    Ok(handle)
 }