From 219e612eca766817d478418006a81ac60e0990a3 Mon Sep 17 00:00:00 2001 From: Maxime Van Hees Date: Tue, 21 Oct 2025 16:37:11 +0200 Subject: [PATCH 1/2] WIP1: implementing JSON-RPC calls over Unix Sockets --- Cargo.lock | 163 ++++++++++++++++++++++++++++++++++++++----- Cargo.toml | 3 +- README.md | 20 +++++- docs/basics.md | 21 +++++- docs/rpc_examples.md | 28 +++++++- src/main.rs | 34 ++++++++- src/rpc.rs | 2 +- src/rpc_server.rs | 34 ++++----- 8 files changed, 265 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66f2f31..1856901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -901,7 +901,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" dependencies = [ "fastrand", - "gloo-timers", + "gloo-timers 0.3.0", "tokio", ] @@ -2310,6 +2310,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "doctest-file" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562" + [[package]] name = "downcast-rs" version = "2.0.2" @@ -2691,6 +2697,10 @@ name = "futures-timer" version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +dependencies = [ + "gloo-timers 0.2.6", + "send_wrapper", +] [[package]] name = "futures-util" @@ -2782,6 +2792,39 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-net" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06f627b1a58ca3d42b45d6104bf1e1a03799df472df00988b6ba21accc10580" +dependencies = [ + "futures-channel", + "futures-core", + "futures-sink", + "gloo-utils", + "http 1.3.1", + "js-sys", + "pin-project", + "serde", + "serde_json", + "thiserror 1.0.69", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "gloo-timers" version = "0.3.0" @@ -2794,6 +2837,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gloo-utils" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5555354113b18c547c1d3a98fbf7fb32a9ff4f6fa112ce823a21641a0ba3aa" +dependencies = [ + "js-sys", + "serde", + "serde_json", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "h2" version = "0.3.27" @@ -2912,6 +2968,7 @@ dependencies = [ "rand 0.8.5", "redb", "redis", + "reth-ipc", "secrecy", "serde", "serde_json", @@ -3400,6 +3457,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "interprocess" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d941b405bd2322993887859a8ee6ac9134945a24ec5ec763a8a962fc64dfec2d" +dependencies = [ + "doctest-file", + "futures-core", + "libc", + "recvmsg", + "tokio", + "widestring", + "windows-sys 0.52.0", +] + [[package]] name = "intl-memoizer" version = "0.5.3" @@ -3587,15 +3659,17 @@ dependencies = [ [[package]] name = "jsonrpsee" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3f48dc3e6b8bd21e15436c1ddd0bc22a6a54e8ec46fedd6adf3425f396ec6a" +checksum = "1fba77a59c4c644fd48732367624d1bcf6f409f9c9a286fbc71d2f1fc0b2ea16" dependencies = [ + "jsonrpsee-client-transport", "jsonrpsee-core", "jsonrpsee-http-client", "jsonrpsee-proc-macros", "jsonrpsee-server", "jsonrpsee-types", + "jsonrpsee-wasm-client", "jsonrpsee-ws-client", "tokio", "tracing", @@ -3603,12 +3677,14 @@ dependencies = [ [[package]] name = "jsonrpsee-client-transport" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf36eb27f8e13fa93dcb50ccb44c417e25b818cfa1a481b5470cd07b19c60b98" +checksum = "a2a320a3f1464e4094f780c4d48413acd786ce5627aaaecfac9e9c7431d13ae1" dependencies = [ "base64 0.22.1", + "futures-channel", "futures-util", + "gloo-net", "http 1.3.1", "jsonrpsee-core", "pin-project", @@ -3626,9 +3702,9 @@ dependencies = [ [[package]] name = "jsonrpsee-core" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "316c96719901f05d1137f19ba598b5fe9c9bc39f4335f67f6be8613921946480" +checksum = "693c93cbb7db25f4108ed121304b671a36002c2db67dff2ee4391a688c738547" dependencies = [ "async-trait", "bytes", @@ -3649,13 +3725,14 @@ dependencies = [ "tokio-stream", "tower", "tracing", + "wasm-bindgen-futures", ] [[package]] name = "jsonrpsee-http-client" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790bedefcec85321e007ff3af84b4e417540d5c87b3c9779b9e247d1bcc3dab8" +checksum = "6962d2bd295f75e97dd328891e58fce166894b974c1f7ce2e7597f02eeceb791" dependencies = [ "base64 0.22.1", "http-body 1.0.1", @@ -3676,9 +3753,9 @@ dependencies = [ [[package]] name = "jsonrpsee-proc-macros" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da3f8ab5ce1bb124b6d082e62dffe997578ceaf0aeb9f3174a214589dc00f07" +checksum = "2fa4f5daed39f982a1bb9d15449a28347490ad42b212f8eaa2a2a344a0dce9e9" dependencies = [ "heck 0.5.0", "proc-macro-crate", @@ -3689,9 +3766,9 @@ dependencies = [ [[package]] name = "jsonrpsee-server" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c51b7c290bb68ce3af2d029648148403863b982f138484a73f02a9dd52dbd7f" +checksum = "d38b0bcf407ac68d241f90e2d46041e6a06988f97fe1721fb80b91c42584fae6" dependencies = [ "futures-util", "http 1.3.1", @@ -3716,9 +3793,9 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc88ff4688e43cc3fa9883a8a95c6fa27aa2e76c96e610b737b6554d650d7fd5" +checksum = "66df7256371c45621b3b7d2fb23aea923d577616b9c0e9c0b950a6ea5c2be0ca" dependencies = [ "http 1.3.1", "serde", @@ -3727,10 +3804,22 @@ dependencies = [ ] [[package]] -name = "jsonrpsee-ws-client" -version = "0.26.0" +name = "jsonrpsee-wasm-client" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6fceceeb05301cc4c065ab3bd2fa990d41ff4eb44e4ca1b30fa99c057c3e79" +checksum = "6b67695cbcf4653f39f8f8738925547e0e23fd9fe315bccf951097b9f6a38781" +dependencies = [ + "jsonrpsee-client-transport", + "jsonrpsee-core", + "jsonrpsee-types", + "tower", +] + +[[package]] +name = "jsonrpsee-ws-client" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da2694c9ff271a9d3ebfe520f6b36820e85133a51be77a3cb549fd615095261" dependencies = [ "http 1.3.1", "jsonrpsee-client-transport", @@ -5440,6 +5529,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "recvmsg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" + [[package]] name = "redb" version = "2.6.2" @@ -5629,6 +5724,26 @@ dependencies = [ "webpki-roots 1.0.2", ] +[[package]] +name = "reth-ipc" +version = "1.6.0" +source = "git+https://github.com/paradigmxyz/reth?rev=d8451e54e7267f9f1634118d6d279b2216f7e2bb#d8451e54e7267f9f1634118d6d279b2216f7e2bb" +dependencies = [ + "bytes", + "futures", + "futures-util", + "interprocess", + "jsonrpsee", + "pin-project", + "serde_json", + "thiserror 2.0.16", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tracing", +] + [[package]] name = "ring" version = "0.17.14" @@ -6078,6 +6193,12 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + [[package]] name = "seq-macro" version = "0.3.6" @@ -7481,6 +7602,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 3f3e0b3..0d441ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ secrecy = "0.8" ed25519-dalek = "2" x25519-dalek = "2" base64 = "0.22" -jsonrpsee = { version = "0.26.0", features = ["http-client", "ws-client", "server", "macros"] } +jsonrpsee = { version = "0.25.1", features = ["http-client", "ws-client", "server", "macros"] } tantivy = "0.25.0" arrow-schema = "55.2.0" arrow-array = "55.2.0" @@ -35,6 +35,7 @@ arrow = "55.2.0" lancedb = "0.22.1" uuid = "1.18.1" ureq = { version = "2.10.0", features = ["json", "tls"] } +reth-ipc = { git = "https://github.com/paradigmxyz/reth", package = "reth-ipc", rev = "d8451e54e7267f9f1634118d6d279b2216f7e2bb" } [dev-dependencies] redis = { version = "0.24", features = ["aio", "tokio-comp"] } diff --git a/README.md b/README.md index f342aba..ebad807 100644 --- a/README.md +++ b/README.md @@ -32,13 +32,31 @@ cargo build --release ### Running HeroDB -Launch HeroDB with the required `--admin-secret` flag, which encrypts the admin database (DB 0) and authorizes admin access. Optional flags include `--dir` for the database directory, `--port` for the TCP port (default 6379), `--sled` for the sled backend, and `--enable-rpc` to start the JSON-RPC management server on port 8080. +Launch HeroDB with the required `--admin-secret` flag, which encrypts the admin database (DB 0) and authorizes admin access. Optional flags include `--dir` for the database directory, `--port` for the TCP port (default 6379), `--sled` for the sled backend, `--enable-rpc` to start the HTTP JSON-RPC server on a TCP port, `--enable-rpc-ipc` to start JSON-RPC over a Unix Domain Socket (non-HTTP), and `--rpc-ipc-path ` to specify the socket path (default: `/tmp/herodb.ipc`). Example: ```bash ./target/release/herodb --dir /tmp/herodb --admin-secret myadminsecret --port 6379 --enable-rpc ``` +To enable JSON-RPC over a Unix Domain Socket at `/tmp/herodb.sock`: +```bash +./target/release/herodb --dir /tmp/herodb --admin-secret myadminsecret --enable-rpc-ipc --rpc-ipc-path /tmp/herodb.sock +``` + +Test the IPC endpoint interactively with socat: +```bash +sudo socat -d -d -t 5 - UNIX-CONNECT:/tmp/herodb.sock +``` +Then paste a framed JSON-RPC request (Content-Length header, blank line, then JSON body). Example: +``` +Content-Length: 73 + +{"jsonrpc":"2.0","method":"hero_listDatabases","params":[],"id":3} +``` + +For a one-liner that auto-computes Content-Length and pretty-prints the JSON response, see docs/rpc_examples.md. + For detailed launch options, see [Basics](docs/basics.md). ## Usage with Redis Clients diff --git a/docs/basics.md b/docs/basics.md index 4d0bbc2..621e29b 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -9,8 +9,10 @@ To launch HeroDB, use the binary with required and optional flags. The `--admin- - `--port `: TCP port for Redis protocol (default: 6379). - `--debug`: Enable debug logging. - `--sled`: Use Sled backend (default: Redb). -- `--enable-rpc`: Start JSON-RPC management server on port 8080. +- `--enable-rpc`: Start JSON-RPC management server on port 8080 (HTTP over TCP). - `--rpc-port `: Custom RPC port (default: 8080). +- `--enable-rpc-ipc`: Start JSON-RPC over a Unix Domain Socket (non-HTTP). +- `--rpc-ipc-path `: Path to the Unix socket for IPC (default: `/tmp/herodb.ipc`). - `--admin-secret `: Required secret for DB 0 encryption and admin access. Example: @@ -18,6 +20,23 @@ Example: ./target/release/herodb --dir /tmp/herodb --admin-secret mysecret --port 6379 --enable-rpc ``` +To enable JSON-RPC over a Unix Domain Socket at `/tmp/herodb.sock`: +```bash +./target/release/herodb --dir /tmp/herodb --admin-secret mysecret --enable-rpc-ipc --rpc-ipc-path /tmp/herodb.sock +``` + +Test the IPC endpoint interactively with socat (non-HTTP transport): +```bash +sudo socat -d -d -t 5 - UNIX-CONNECT:/tmp/herodb.sock +``` +Then paste a framed JSON-RPC request (Content-Length header, blank line, then JSON body). Example: +``` +Content-Length: 73 + +{"jsonrpc":"2.0","method":"hero_listDatabases","params":[],"id":3} +``` +More IPC examples are in [docs/rpc_examples.md](docs/rpc_examples.md). + Deprecated flags (`--encrypt`, `--encryption-key`) are ignored for data DBs; per-database encryption is managed via RPC. ## Admin Database (DB 0) diff --git a/docs/rpc_examples.md b/docs/rpc_examples.md index 2b941cc..2e5e86d 100644 --- a/docs/rpc_examples.md +++ b/docs/rpc_examples.md @@ -138,4 +138,30 @@ Returns stats like total databases and uptime. - Per-database encryption keys are write-only; set at creation and used transparently. - Access keys are hashed (SHA-256) for storage; provide plaintext in requests. - Backend options: `"Redb"` (default) or `"Sled"`. -- Config object fields (name, storage_path, etc.) are optional and currently ignored but positional. \ No newline at end of file +- Config object fields (name, storage_path, etc.) are optional and currently ignored but positional. + +## IPC over Unix Socket (non-HTTP) + +HeroDB supports JSON-RPC over a Unix Domain Socket using reth-ipc. This transport is not HTTP; messages are JSON-RPC framed with a Content-Length header. + +- Enable IPC on startup (adjust the socket path as needed): + - herodb --dir /path/to/data --admin-secret YOUR_SECRET --enable-rpc-ipc --rpc-ipc-path /tmp/herodb.sock + +- The same RPC methods are available as over HTTP. Namespace is "hero" (e.g. hero_listDatabases). See the RPC trait in [src/rpc.rs](src/rpc.rs) and CLI flags in [src/main.rs](src/main.rs). The IPC bootstrap is in [src/rpc_server.rs](src/rpc_server.rs). + +### Test via socat (interactive) + +1) Connect to the socket with a small timeout: +``` +sudo socat -d -d -t 5 - UNIX-CONNECT:/tmp/herodb.sock +``` + +2) Paste a framed JSON-RPC request (Content-Length header, then a blank line, then the JSON body). For example to call hero_listDatabases: + +Content-Length: <LEN-BYTES-OF-JSON> + +{"jsonrpc":"2.0","id":3,"method":"hero_listDatabases","params":[]} + +Notes: +- Replace <LEN-BYTES-OF-JSON> with the byte-length of the exact JSON payload you paste. There must be an empty line between the header and the JSON. +- The response will appear in the same terminal, also framed with Content-Length. \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index d5bacba..2f004c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,6 +40,14 @@ struct Args { #[arg(long, default_value = "8080")] rpc_port: u16, + /// Enable RPC over Unix Domain Socket (IPC) + #[arg(long)] + enable_rpc_ipc: bool, + + /// RPC IPC socket path (Unix Domain Socket) + #[arg(long, default_value = "/tmp/herodb.ipc")] + rpc_ipc_path: String, + /// Use the sled backend #[arg(long)] sled: bool, @@ -105,7 +113,7 @@ async fn main() { let rpc_addr = format!("127.0.0.1:{}", args.rpc_port).parse().unwrap(); let base_dir = args.dir.clone(); - match rpc_server::start_rpc_server(rpc_addr, base_dir, backend, args.admin_secret.clone()).await { + match rpc_server::start_rpc_server(rpc_addr, base_dir, backend.clone(), args.admin_secret.clone()).await { Ok(handle) => { println!("RPC management server started on port {}", args.rpc_port); Some(handle) @@ -119,6 +127,30 @@ async fn main() { None }; + // Start IPC (Unix socket) RPC server if enabled + let _rpc_ipc_handle = if args.enable_rpc_ipc { + let base_dir = args.dir.clone(); + let ipc_path = args.rpc_ipc_path.clone(); + + // Remove stale socket if present + if std::path::Path::new(&ipc_path).exists() { + let _ = std::fs::remove_file(&ipc_path); + } + + match rpc_server::start_rpc_ipc_server(ipc_path.clone(), base_dir, backend.clone(), args.admin_secret.clone()).await { + Ok(handle) => { + println!("RPC IPC server started at {}", ipc_path); + Some(handle) + } + Err(e) => { + eprintln!("Failed to start RPC IPC server: {}", e); + None + } + } + } else { + None + }; + // accept new connections loop { let stream = listener.accept().await; diff --git a/src/rpc.rs b/src/rpc.rs index b8a1cf2..90d6d06 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -71,7 +71,7 @@ pub fn hash_key(key: &str) -> String { } /// RPC trait for HeroDB management -#[rpc(server, client, namespace = "hero")] +#[rpc(server, client, namespace = "herodb")] pub trait Rpc { /// Create a new database with specified configuration #[method(name = "createDatabase")] diff --git a/src/rpc_server.rs b/src/rpc_server.rs index 8000833..17f855a 100644 --- a/src/rpc_server.rs +++ b/src/rpc_server.rs @@ -2,6 +2,7 @@ use std::net::SocketAddr; use std::path::PathBuf; use jsonrpsee::server::{ServerBuilder, ServerHandle}; use jsonrpsee::RpcModule; +use reth_ipc::server::Builder as IpcServerBuilder; use crate::rpc::{RpcServer, RpcServerImpl}; @@ -27,24 +28,25 @@ pub async fn start_rpc_server(addr: SocketAddr, base_dir: PathBuf, backend: crat Ok(handle) } -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; +/// Start the JSON-RPC IPC server on the specified Unix socket endpoint +pub async fn start_rpc_ipc_server( + endpoint: String, + base_dir: PathBuf, + backend: crate::options::BackendType, + admin_secret: String, +) -> Result> { + // Create the RPC server implementation + let rpc_impl = RpcServerImpl::new(base_dir, backend, admin_secret); - #[tokio::test] - async fn test_rpc_server_startup() { - let addr = "127.0.0.1:0".parse().unwrap(); // Use port 0 for auto-assignment - let base_dir = PathBuf::from("/tmp/test_rpc"); - let backend = crate::options::BackendType::Redb; // Default for test + // Create the RPC module + let mut module = RpcModule::new(()); + module.merge(RpcServer::into_rpc(rpc_impl))?; - let handle = start_rpc_server(addr, base_dir, backend, "test-admin".to_string()).await.unwrap(); + // Build the IPC server and start it + let server = IpcServerBuilder::default().build(endpoint.clone()); + let handle = server.start(module).await?; - // Give the server a moment to start - tokio::time::sleep(Duration::from_millis(100)).await; + println!("RPC IPC server started on {}", endpoint); - // Stop the server - handle.stop().unwrap(); - handle.stopped().await; - } + Ok(handle) } \ No newline at end of file From 6ae5d6f4f93ec07f8292d20780ec52acc8b5603f Mon Sep 17 00:00:00 2001 From: Maxime Van Hees Date: Thu, 23 Oct 2025 15:42:05 +0200 Subject: [PATCH 2/2] testing RPC over Unix socket + updated docs --- docs/basics.md | 4 +--- docs/rpc_examples.md | 8 +------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/basics.md b/docs/basics.md index 621e29b..bee4c4a 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -29,10 +29,8 @@ Test the IPC endpoint interactively with socat (non-HTTP transport): ```bash sudo socat -d -d -t 5 - UNIX-CONNECT:/tmp/herodb.sock ``` -Then paste a framed JSON-RPC request (Content-Length header, blank line, then JSON body). Example: +Then paste a framed JSON-RPC request. Example: ``` -Content-Length: 73 - {"jsonrpc":"2.0","method":"hero_listDatabases","params":[],"id":3} ``` More IPC examples are in [docs/rpc_examples.md](docs/rpc_examples.md). diff --git a/docs/rpc_examples.md b/docs/rpc_examples.md index 2e5e86d..96d280f 100644 --- a/docs/rpc_examples.md +++ b/docs/rpc_examples.md @@ -158,10 +158,4 @@ sudo socat -d -d -t 5 - UNIX-CONNECT:/tmp/herodb.sock 2) Paste a framed JSON-RPC request (Content-Length header, then a blank line, then the JSON body). For example to call hero_listDatabases: -Content-Length: <LEN-BYTES-OF-JSON> - -{"jsonrpc":"2.0","id":3,"method":"hero_listDatabases","params":[]} - -Notes: -- Replace <LEN-BYTES-OF-JSON> with the byte-length of the exact JSON payload you paste. There must be an empty line between the header and the JSON. -- The response will appear in the same terminal, also framed with Content-Length. \ No newline at end of file +{"jsonrpc":"2.0","id":3,"method":"hero_listDatabases","params":[]} \ No newline at end of file