From 9fa983260597ae6e29a661379cf2b137a27a6115 Mon Sep 17 00:00:00 2001 From: Maxime Van Hees Date: Thu, 11 Sep 2025 17:23:46 +0200 Subject: [PATCH 1/6] combined curret main (with sled) and RPC server --- Cargo.lock | 926 ++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 1 + src/lib.rs | 2 + src/main.rs | 32 +- src/rpc.rs | 335 ++++++++++++++++ src/rpc_server.rs | 59 +++ tests/rpc_tests.rs | 62 +++ 7 files changed, 1396 insertions(+), 21 deletions(-) create mode 100644 src/rpc.rs create mode 100644 src/rpc_server.rs create mode 100644 tests/rpc_tests.rs diff --git a/Cargo.lock b/Cargo.lock index e0ec8c8..c3faa26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,7 +44,7 @@ dependencies = [ "lazy_static", "nom", "pin-project", - "rand", + "rand 0.8.5", "rust-embed", "scrypt", "sha2", @@ -65,7 +65,7 @@ dependencies = [ "hkdf", "io_tee", "nom", - "rand", + "rand 0.8.5", "secrecy", "sha2", ] @@ -143,6 +143,12 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -239,6 +245,22 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "cc" +version = "1.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.3" @@ -355,6 +377,22 @@ dependencies = [ "futures", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -395,7 +433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "typenum", ] @@ -433,7 +471,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core 0.9.11", @@ -495,6 +533,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -510,6 +554,12 @@ dependencies = [ "toml", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" + [[package]] name = "fluent" version = "0.16.1" @@ -551,9 +601,15 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a530c4694a6a8d528794ee9bbd8ba0122e779629ac908d15ad5a7ae7763a33d" dependencies = [ - "thiserror", + "thiserror 1.0.69", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -644,6 +700,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -689,7 +751,19 @@ checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.5+wasi-0.2.4", ] [[package]] @@ -698,12 +772,37 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" + [[package]] name = "heck" version = "0.5.0" @@ -724,7 +823,8 @@ dependencies = [ "clap", "ed25519-dalek", "futures", - "rand", + "jsonrpsee", + "rand 0.8.5", "redb", "redis", "secrecy", @@ -732,7 +832,7 @@ dependencies = [ "serde_json", "sha2", "sled", - "thiserror", + "thiserror 1.0.69", "tokio", ] @@ -754,6 +854,113 @@ dependencies = [ "digest", ] +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2 0.6.0", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "i18n-config" version = "0.4.8" @@ -764,7 +971,7 @@ dependencies = [ "log", "serde", "serde_derive", - "thiserror", + "thiserror 1.0.69", "unic-langid", ] @@ -784,7 +991,7 @@ dependencies = [ "log", "parking_lot 0.12.4", "rust-embed", - "thiserror", + "thiserror 1.0.69", "unic-langid", "walkdir", ] @@ -930,6 +1137,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921" +dependencies = [ + "equivalent", + "hashbrown 0.15.5", +] + [[package]] name = "inout" version = "0.1.4" @@ -996,6 +1213,183 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jsonrpsee" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3f48dc3e6b8bd21e15436c1ddd0bc22a6a54e8ec46fedd6adf3425f396ec6a" +dependencies = [ + "jsonrpsee-core", + "jsonrpsee-http-client", + "jsonrpsee-proc-macros", + "jsonrpsee-server", + "jsonrpsee-types", + "jsonrpsee-ws-client", + "tokio", + "tracing", +] + +[[package]] +name = "jsonrpsee-client-transport" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf36eb27f8e13fa93dcb50ccb44c417e25b818cfa1a481b5470cd07b19c60b98" +dependencies = [ + "base64 0.22.1", + "futures-util", + "http", + "jsonrpsee-core", + "pin-project", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "soketto", + "thiserror 2.0.16", + "tokio", + "tokio-rustls", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "jsonrpsee-core" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "316c96719901f05d1137f19ba598b5fe9c9bc39f4335f67f6be8613921946480" +dependencies = [ + "async-trait", + "bytes", + "futures-timer", + "futures-util", + "http", + "http-body", + "http-body-util", + "jsonrpsee-types", + "parking_lot 0.12.4", + "pin-project", + "rand 0.9.2", + "rustc-hash 2.1.1", + "serde", + "serde_json", + "thiserror 2.0.16", + "tokio", + "tokio-stream", + "tower", + "tracing", +] + +[[package]] +name = "jsonrpsee-http-client" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790bedefcec85321e007ff3af84b4e417540d5c87b3c9779b9e247d1bcc3dab8" +dependencies = [ + "base64 0.22.1", + "http-body", + "hyper", + "hyper-rustls", + "hyper-util", + "jsonrpsee-core", + "jsonrpsee-types", + "rustls", + "rustls-platform-verifier", + "serde", + "serde_json", + "thiserror 2.0.16", + "tokio", + "tower", + "url", +] + +[[package]] +name = "jsonrpsee-proc-macros" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da3f8ab5ce1bb124b6d082e62dffe997578ceaf0aeb9f3174a214589dc00f07" +dependencies = [ + "heck", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "jsonrpsee-server" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c51b7c290bb68ce3af2d029648148403863b982f138484a73f02a9dd52dbd7f" +dependencies = [ + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "jsonrpsee-core", + "jsonrpsee-types", + "pin-project", + "route-recognizer", + "serde", + "serde_json", + "soketto", + "thiserror 2.0.16", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tracing", +] + +[[package]] +name = "jsonrpsee-types" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc88ff4688e43cc3fa9883a8a95c6fa27aa2e76c96e610b737b6554d650d7fd5" +dependencies = [ + "http", + "serde", + "serde_json", + "thiserror 2.0.16", +] + +[[package]] +name = "jsonrpsee-ws-client" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6fceceeb05301cc4c065ab3bd2fa990d41ff4eb44e4ca1b30fa99c057c3e79" +dependencies = [ + "http", + "jsonrpsee-client-transport", + "jsonrpsee-core", + "jsonrpsee-types", + "tower", + "url", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1058,7 +1452,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.59.0", ] @@ -1099,6 +1493,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + [[package]] name = "parking_lot" version = "0.11.2" @@ -1234,6 +1634,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1276,6 +1685,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rand" version = "0.8.5" @@ -1283,8 +1698,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -1294,7 +1719,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -1303,7 +1738,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", ] [[package]] @@ -1354,6 +1798,26 @@ dependencies = [ "bitflags 2.9.3", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "route-recognizer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" + [[package]] name = "rust-embed" version = "8.7.2" @@ -1415,6 +1879,80 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.23.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs 0.26.11", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a37813727b78798e53c2bec3f5e8fe12a6d6f8389bf9ca7802add4c9905ad8" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.20" @@ -1439,6 +1977,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1465,6 +2012,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b369d18893388b345804dc0007963c99b7d665ae71d275812d828c6f089640" +dependencies = [ + "bitflags 2.9.3", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "self_cell" version = "0.10.3" @@ -1518,6 +2088,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -1535,6 +2116,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.6" @@ -1550,7 +2137,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -1601,6 +2188,22 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "soketto" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e859df029d160cb88608f5d7df7fb4753fd20fdfb4de5644f3d8b8440841721" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures", + "http", + "httparse", + "log", + "rand 0.8.5", + "sha1", +] + [[package]] name = "spki" version = "0.7.3" @@ -1656,6 +2259,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.2" @@ -1673,7 +2282,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +dependencies = [ + "thiserror-impl 2.0.16", ] [[package]] @@ -1687,6 +2305,17 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "thiserror-impl" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "tinystr" version = "0.8.1" @@ -1728,6 +2357,28 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-util" version = "0.7.16" @@ -1736,6 +2387,7 @@ checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -1750,6 +2402,86 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "type-map" version = "0.5.1" @@ -1800,6 +2532,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.6" @@ -1839,12 +2577,57 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.5+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4494f6290a82f5fe584817a676a34b9d6763e8d9d18204009fb31dceca98fd4" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.0+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03fa2761397e5bd52002cd7e73110c71af2109aca4e521a9f40473fe685b0a24" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" +dependencies = [ + "webpki-root-certs 1.0.2", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4ffd8df1c57e87c325000a3d6ef93db75279dc3a231125aac571650f22b12a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1882,6 +2665,30 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -1900,6 +2707,30 @@ dependencies = [ "windows-targets 0.53.3", ] +[[package]] +name = "windows-sys" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +dependencies = [ + "windows-link 0.2.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1922,7 +2753,7 @@ version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ - "windows-link", + "windows-link 0.1.3", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", @@ -1933,6 +2764,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -1945,6 +2782,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -1957,6 +2800,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1981,6 +2830,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -1993,6 +2848,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -2005,6 +2866,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -2017,6 +2884,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -2029,6 +2902,21 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winnow" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" + [[package]] name = "writeable" version = "0.6.1" @@ -2042,7 +2930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ "curve25519-dalek", - "rand_core", + "rand_core 0.6.4", "serde", "zeroize", ] diff --git a/Cargo.toml b/Cargo.toml index 50d3208..4ba1e29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ age = "0.10" secrecy = "0.8" ed25519-dalek = "2" base64 = "0.22" +jsonrpsee = { version = "0.26.0", features = ["http-client", "ws-client", "server", "macros"] } [dev-dependencies] redis = { version = "0.24", features = ["aio", "tokio-comp"] } diff --git a/src/lib.rs b/src/lib.rs index 31e69a8..85dd4e4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,8 @@ pub mod crypto; pub mod error; pub mod options; pub mod protocol; +pub mod rpc; +pub mod rpc_server; pub mod server; pub mod storage; pub mod storage_trait; // Add this diff --git a/src/main.rs b/src/main.rs index dce569b..34d7fcf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ use tokio::net::TcpListener; use herodb::server; +use herodb::rpc_server; use clap::Parser; @@ -31,6 +32,14 @@ struct Args { #[arg(long)] encrypt: bool, + /// Enable RPC management server + #[arg(long)] + enable_rpc: bool, + + /// RPC server port (default: 8080) + #[arg(long, default_value = "8080")] + rpc_port: u16, + /// Use the sled backend #[arg(long)] sled: bool, @@ -50,7 +59,7 @@ async fn main() { // new DB option let option = herodb::options::DBOption { - dir: args.dir, + dir: args.dir.clone(), port, debug: args.debug, encryption_key: args.encryption_key, @@ -63,11 +72,30 @@ async fn main() { }; // new server - let server = server::Server::new(option).await; + let server = server::Server::new(option.clone()).await; // Add a small delay to ensure the port is ready tokio::time::sleep(std::time::Duration::from_millis(100)).await; + // Start RPC server if enabled + let rpc_handle = if args.enable_rpc { + let rpc_addr = format!("127.0.0.1:{}", args.rpc_port).parse().unwrap(); + let base_dir = args.dir.clone(); + + match rpc_server::start_rpc_server(rpc_addr, base_dir, option).await { + Ok(handle) => { + println!("RPC management server started on port {}", args.rpc_port); + Some(handle) + } + Err(e) => { + eprintln!("Failed to start RPC server: {}", e); + None + } + } + } else { + None + }; + // accept new connections loop { let stream = listener.accept().await; diff --git a/src/rpc.rs b/src/rpc.rs new file mode 100644 index 0000000..4715767 --- /dev/null +++ b/src/rpc.rs @@ -0,0 +1,335 @@ +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::RwLock; +use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use serde::{Deserialize, Serialize}; + +use crate::server::Server; +use crate::options::DBOption; + +/// Database backend types +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum BackendType { + Redb, + Sled, + // Future: InMemory, Custom(String) +} + +/// Database configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DatabaseConfig { + pub name: Option, + pub storage_path: Option, + pub max_size: Option, + pub redis_version: Option, +} + +/// Database information returned by metadata queries +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DatabaseInfo { + pub id: u64, + pub name: Option, + pub backend: BackendType, + pub encrypted: bool, + pub redis_version: Option, + pub storage_path: Option, + pub size_on_disk: Option, + pub key_count: Option, + pub created_at: u64, + pub last_access: Option, +} + +/// RPC trait for HeroDB management +#[rpc(server, client, namespace = "herodb")] +pub trait Rpc { + /// Create a new database with specified configuration + #[method(name = "createDatabase")] + async fn create_database( + &self, + backend: BackendType, + config: DatabaseConfig, + encryption_key: Option, + ) -> RpcResult; + + /// Set encryption for an existing database (write-only key) + #[method(name = "setEncryption")] + async fn set_encryption(&self, db_id: u64, encryption_key: String) -> RpcResult; + + /// List all managed databases + #[method(name = "listDatabases")] + async fn list_databases(&self) -> RpcResult>; + + /// Get detailed information about a specific database + #[method(name = "getDatabaseInfo")] + async fn get_database_info(&self, db_id: u64) -> RpcResult; + + /// Delete a database + #[method(name = "deleteDatabase")] + async fn delete_database(&self, db_id: u64) -> RpcResult; + + /// Get server statistics + #[method(name = "getServerStats")] + async fn get_server_stats(&self) -> RpcResult>; +} + +/// RPC Server implementation +pub struct RpcServerImpl { + /// Base directory for database files + base_dir: String, + /// Managed database servers + servers: Arc>>>, + /// Next database ID to assign + next_db_id: Arc>, + /// Database options (backend, encryption, etc.) + db_option: DBOption, +} + +impl RpcServerImpl { + /// Create a new RPC server instance + pub fn new(base_dir: String, db_option: DBOption) -> Self { + Self { + base_dir, + servers: Arc::new(RwLock::new(HashMap::new())), + next_db_id: Arc::new(RwLock::new(0)), + db_option, + } + } + + /// Get or create a server instance for the given database ID + async fn get_or_create_server(&self, db_id: u64) -> Result, jsonrpsee::types::ErrorObjectOwned> { + // Check if server already exists + { + let servers = self.servers.read().await; + if let Some(server) = servers.get(&db_id) { + return Ok(server.clone()); + } + } + + // Check if database file exists (either direct or in RPC subdirectory) + let direct_db_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}.db", db_id)); + let rpc_db_path = std::path::PathBuf::from(&self.base_dir) + .join(format!("rpc_db_{}", db_id)) + .join("0.db"); + + let (db_path, db_dir) = if direct_db_path.exists() { + // Main server database + (direct_db_path, self.base_dir.clone()) + } else if rpc_db_path.exists() { + // RPC database + (rpc_db_path, std::path::PathBuf::from(&self.base_dir).join(format!("rpc_db_{}", db_id)).to_string_lossy().to_string()) + } else { + return Err(jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Database {} not found", db_id), + None::<()> + )); + }; + + // Create server instance + let mut db_option = self.db_option.clone(); + db_option.dir = db_dir; + + let server = Server::new(db_option).await; + + // Store the server + let mut servers = self.servers.write().await; + servers.insert(db_id, Arc::new(server.clone())); + + Ok(Arc::new(server)) + } + + /// Discover existing database files in the base directory and RPC subdirectories + async fn discover_databases(&self) -> Vec { + let mut db_ids = Vec::new(); + + if let Ok(entries) = std::fs::read_dir(&self.base_dir) { + for entry in entries.flatten() { + let path = entry.path(); + + // Check if it's a directory starting with "rpc_db_" + if path.is_dir() { + if let Some(dir_name) = path.file_name().and_then(|n| n.to_str()) { + if dir_name.starts_with("rpc_db_") { + // Extract database ID from directory name (e.g., "rpc_db_1" -> 1) + if let Some(id_str) = dir_name.strip_prefix("rpc_db_") { + if let Ok(db_id) = id_str.parse::() { + db_ids.push(db_id); + } + } + } + } + } + // Also check for direct .db files (for main server databases) + else if let Some(file_name) = entry.file_name().to_str() { + if file_name.ends_with(".db") { + // Extract database ID from filename (e.g., "11.db" -> 11) + if let Some(id_str) = file_name.strip_suffix(".db") { + if let Ok(db_id) = id_str.parse::() { + db_ids.push(db_id); + } + } + } + } + } + } + + db_ids + } + + /// Get the next available database ID + async fn get_next_db_id(&self) -> u64 { + let mut id = self.next_db_id.write().await; + let current_id = *id; + *id += 1; + current_id + } +} + +#[jsonrpsee::core::async_trait] +impl RpcServer for RpcServerImpl { + async fn create_database( + &self, + backend: BackendType, + config: DatabaseConfig, + encryption_key: Option, + ) -> RpcResult { + let db_id = self.get_next_db_id().await; + + // Handle both Redb and Sled backends + match backend { + BackendType::Redb | BackendType::Sled => { + // Always create RPC databases in subdirectories to avoid conflicts with main server + let db_dir = if let Some(path) = &config.storage_path { + std::path::PathBuf::from(path).join(format!("rpc_db_{}", db_id)) + } else { + std::path::PathBuf::from(&self.base_dir).join(format!("rpc_db_{}", db_id)) + }; + + // Ensure directory exists + std::fs::create_dir_all(&db_dir) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to create directory: {}", e), + None::<()> + ))?; + + // Create DB options + let encrypt = encryption_key.is_some(); + let option = DBOption { + dir: db_dir.to_string_lossy().to_string(), + port: 0, // Not used for RPC-managed databases + debug: false, + encryption_key, + encrypt, + backend: match backend { + BackendType::Redb => crate::options::BackendType::Redb, + BackendType::Sled => crate::options::BackendType::Sled, + }, + }; + + // Create server instance + let server = Server::new(option).await; + + // Store the server + let mut servers = self.servers.write().await; + servers.insert(db_id, Arc::new(server)); + + Ok(db_id) + } + } + } + + async fn set_encryption(&self, db_id: u64, _encryption_key: String) -> RpcResult { + // Note: In a real implementation, we'd need to modify the existing database + // For now, return false as encryption can only be set during creation + let _servers = self.servers.read().await; + // TODO: Implement encryption setting for existing databases + Ok(false) + } + + async fn list_databases(&self) -> RpcResult> { + let db_ids = self.discover_databases().await; + let mut result = Vec::new(); + + for db_id in db_ids { + // Try to get or create server for this database + if let Ok(server) = self.get_or_create_server(db_id).await { + let backend = match server.option.backend { + crate::options::BackendType::Redb => BackendType::Redb, + crate::options::BackendType::Sled => BackendType::Sled, + }; + + let info = DatabaseInfo { + id: db_id, + name: None, // TODO: Store name in server metadata + backend, + encrypted: server.option.encrypt, + redis_version: Some("7.0".to_string()), // Default Redis compatibility + storage_path: Some(server.option.dir.clone()), + size_on_disk: None, // TODO: Calculate actual size + key_count: None, // TODO: Get key count from storage + created_at: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), + last_access: None, + }; + result.push(info); + } + } + + Ok(result) + } + + async fn get_database_info(&self, db_id: u64) -> RpcResult { + let server = self.get_or_create_server(db_id).await?; + + let backend = match server.option.backend { + crate::options::BackendType::Redb => BackendType::Redb, + crate::options::BackendType::Sled => BackendType::Sled, + }; + + Ok(DatabaseInfo { + id: db_id, + name: None, + backend, + encrypted: server.option.encrypt, + redis_version: Some("7.0".to_string()), + storage_path: Some(server.option.dir.clone()), + size_on_disk: None, + key_count: None, + created_at: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), + last_access: None, + }) + } + + async fn delete_database(&self, db_id: u64) -> RpcResult { + let mut servers = self.servers.write().await; + + if let Some(server) = servers.remove(&db_id) { + // TODO: Clean up database files + let _ = server; + Ok(true) + } else { + Ok(false) + } + } + + async fn get_server_stats(&self) -> RpcResult> { + let db_ids = self.discover_databases().await; + let mut stats = HashMap::new(); + + stats.insert("total_databases".to_string(), serde_json::json!(db_ids.len())); + stats.insert("uptime".to_string(), serde_json::json!( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + )); + + Ok(stats) + } +} \ No newline at end of file diff --git a/src/rpc_server.rs b/src/rpc_server.rs new file mode 100644 index 0000000..a8e9048 --- /dev/null +++ b/src/rpc_server.rs @@ -0,0 +1,59 @@ +use std::net::SocketAddr; +use jsonrpsee::server::{ServerBuilder, ServerHandle}; +use jsonrpsee::RpcModule; + +use crate::rpc::{RpcServer, RpcServerImpl}; +use crate::options::DBOption; + +/// Start the RPC server on the specified address +pub async fn start_rpc_server(addr: SocketAddr, base_dir: String, db_option: DBOption) -> Result> { + // Create the RPC server implementation + let rpc_impl = RpcServerImpl::new(base_dir, db_option); + + // Create the RPC module + let mut module = RpcModule::new(()); + module.merge(RpcServer::into_rpc(rpc_impl))?; + + // Build the server with both HTTP and WebSocket support + let server = ServerBuilder::default() + .build(addr) + .await?; + + // Start the server + let handle = server.start(module); + + println!("RPC server started on {}", addr); + + Ok(handle) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[tokio::test] + async fn test_rpc_server_startup() { + let addr = "127.0.0.1:0".parse().unwrap(); // Use port 0 for auto-assignment + let base_dir = "/tmp/test_rpc".to_string(); + + // Create a dummy DBOption for testing + let db_option = crate::options::DBOption { + dir: base_dir.clone(), + port: 0, + debug: false, + encryption_key: None, + encrypt: false, + backend: crate::options::BackendType::Redb, + }; + + let handle = start_rpc_server(addr, base_dir, db_option).await.unwrap(); + + // Give the server a moment to start + tokio::time::sleep(Duration::from_millis(100)).await; + + // Stop the server + handle.stop().unwrap(); + handle.stopped().await; + } +} \ No newline at end of file diff --git a/tests/rpc_tests.rs b/tests/rpc_tests.rs new file mode 100644 index 0000000..741cf70 --- /dev/null +++ b/tests/rpc_tests.rs @@ -0,0 +1,62 @@ +use std::net::SocketAddr; +use jsonrpsee::http_client::HttpClientBuilder; +use jsonrpsee::core::client::ClientT; +use serde_json::json; + +use herodb::rpc::{RpcClient, BackendType, DatabaseConfig}; + +#[tokio::test] +async fn test_rpc_server_basic() { + // This test would require starting the RPC server in a separate thread + // For now, we'll just test that the types compile correctly + + // Test serialization of types + let backend = BackendType::Redb; + let config = DatabaseConfig { + name: Some("test_db".to_string()), + storage_path: Some("/tmp/test".to_string()), + max_size: Some(1024 * 1024), + redis_version: Some("7.0".to_string()), + }; + + let backend_json = serde_json::to_string(&backend).unwrap(); + let config_json = serde_json::to_string(&config).unwrap(); + + assert_eq!(backend_json, "\"Redb\""); + assert!(config_json.contains("test_db")); +} + +#[tokio::test] +async fn test_database_config_serialization() { + let config = DatabaseConfig { + name: Some("my_db".to_string()), + storage_path: None, + max_size: Some(1000000), + redis_version: Some("7.0".to_string()), + }; + + let json = serde_json::to_value(&config).unwrap(); + assert_eq!(json["name"], "my_db"); + assert_eq!(json["max_size"], 1000000); + assert_eq!(json["redis_version"], "7.0"); +} + +#[tokio::test] +async fn test_backend_type_serialization() { + // Test that both Redb and Sled backends serialize correctly + let redb_backend = BackendType::Redb; + let sled_backend = BackendType::Sled; + + let redb_json = serde_json::to_string(&redb_backend).unwrap(); + let sled_json = serde_json::to_string(&sled_backend).unwrap(); + + assert_eq!(redb_json, "\"Redb\""); + assert_eq!(sled_json, "\"Sled\""); + + // Test deserialization + let redb_deserialized: BackendType = serde_json::from_str(&redb_json).unwrap(); + let sled_deserialized: BackendType = serde_json::from_str(&sled_json).unwrap(); + + assert!(matches!(redb_deserialized, BackendType::Redb)); + assert!(matches!(sled_deserialized, BackendType::Sled)); +} \ No newline at end of file -- 2.43.0 From 8798bc202eb944653f9be4639f92bece8e5fba37 Mon Sep 17 00:00:00 2001 From: Maxime Van Hees Date: Thu, 11 Sep 2025 18:33:09 +0200 Subject: [PATCH 2/6] Restore working code --- src/main.rs | 9 +++- src/rpc.rs | 115 ++++++++++++++++++++++++---------------------- src/rpc_server.rs | 18 ++------ 3 files changed, 72 insertions(+), 70 deletions(-) diff --git a/src/main.rs b/src/main.rs index 34d7fcf..6b88c33 100644 --- a/src/main.rs +++ b/src/main.rs @@ -71,8 +71,13 @@ async fn main() { }, }; + let backend = option.backend.clone(); + // new server - let server = server::Server::new(option.clone()).await; + let mut server = server::Server::new(option).await; + + // Initialize the default database storage + let _ = server.current_storage(); // Add a small delay to ensure the port is ready tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -82,7 +87,7 @@ async fn main() { let rpc_addr = format!("127.0.0.1:{}", args.rpc_port).parse().unwrap(); let base_dir = args.dir.clone(); - match rpc_server::start_rpc_server(rpc_addr, base_dir, option).await { + match rpc_server::start_rpc_server(rpc_addr, base_dir, backend).await { Ok(handle) => { println!("RPC management server started on port {}", args.rpc_port); Some(handle) diff --git a/src/rpc.rs b/src/rpc.rs index 4715767..b791bec 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -78,20 +78,23 @@ pub struct RpcServerImpl { base_dir: String, /// Managed database servers servers: Arc>>>, - /// Next database ID to assign - next_db_id: Arc>, - /// Database options (backend, encryption, etc.) - db_option: DBOption, + /// Next unencrypted database ID to assign + next_unencrypted_id: Arc>, + /// Next encrypted database ID to assign + next_encrypted_id: Arc>, + /// Default backend type + backend: crate::options::BackendType, } impl RpcServerImpl { /// Create a new RPC server instance - pub fn new(base_dir: String, db_option: DBOption) -> Self { + pub fn new(base_dir: String, backend: crate::options::BackendType) -> Self { Self { base_dir, servers: Arc::new(RwLock::new(HashMap::new())), - next_db_id: Arc::new(RwLock::new(0)), - db_option, + next_unencrypted_id: Arc::new(RwLock::new(0)), + next_encrypted_id: Arc::new(RwLock::new(10)), + backend, } } @@ -105,31 +108,30 @@ impl RpcServerImpl { } } - // Check if database file exists (either direct or in RPC subdirectory) - let direct_db_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}.db", db_id)); - let rpc_db_path = std::path::PathBuf::from(&self.base_dir) - .join(format!("rpc_db_{}", db_id)) - .join("0.db"); - - let (db_path, db_dir) = if direct_db_path.exists() { - // Main server database - (direct_db_path, self.base_dir.clone()) - } else if rpc_db_path.exists() { - // RPC database - (rpc_db_path, std::path::PathBuf::from(&self.base_dir).join(format!("rpc_db_{}", db_id)).to_string_lossy().to_string()) - } else { + // Check if database file exists + let db_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}.db", db_id)); + if !db_path.exists() { return Err(jsonrpsee::types::ErrorObjectOwned::owned( -32000, format!("Database {} not found", db_id), None::<()> )); + } + + // Create server instance with default options + let db_option = DBOption { + dir: self.base_dir.clone(), + port: 0, // Not used for RPC-managed databases + debug: false, + encryption_key: None, + encrypt: false, + backend: self.backend.clone(), }; - // Create server instance - let mut db_option = self.db_option.clone(); - db_option.dir = db_dir; + let mut server = Server::new(db_option).await; - let server = Server::new(db_option).await; + // Set the selected database to the db_id for proper file naming + server.selected_db = db_id; // Store the server let mut servers = self.servers.write().await; @@ -138,29 +140,14 @@ impl RpcServerImpl { Ok(Arc::new(server)) } - /// Discover existing database files in the base directory and RPC subdirectories + /// Discover existing database files in the base directory async fn discover_databases(&self) -> Vec { let mut db_ids = Vec::new(); if let Ok(entries) = std::fs::read_dir(&self.base_dir) { for entry in entries.flatten() { - let path = entry.path(); - - // Check if it's a directory starting with "rpc_db_" - if path.is_dir() { - if let Some(dir_name) = path.file_name().and_then(|n| n.to_str()) { - if dir_name.starts_with("rpc_db_") { - // Extract database ID from directory name (e.g., "rpc_db_1" -> 1) - if let Some(id_str) = dir_name.strip_prefix("rpc_db_") { - if let Ok(db_id) = id_str.parse::() { - db_ids.push(db_id); - } - } - } - } - } - // Also check for direct .db files (for main server databases) - else if let Some(file_name) = entry.file_name().to_str() { + if let Ok(file_name) = entry.file_name().into_string() { + // Check if it's a database file (ends with .db) if file_name.ends_with(".db") { // Extract database ID from filename (e.g., "11.db" -> 11) if let Some(id_str) = file_name.strip_suffix(".db") { @@ -177,11 +164,18 @@ impl RpcServerImpl { } /// Get the next available database ID - async fn get_next_db_id(&self) -> u64 { - let mut id = self.next_db_id.write().await; - let current_id = *id; - *id += 1; - current_id + async fn get_next_db_id(&self, is_encrypted: bool) -> u64 { + if is_encrypted { + let mut id = self.next_encrypted_id.write().await; + let current_id = *id; + *id += 1; + current_id + } else { + let mut id = self.next_unencrypted_id.write().await; + let current_id = *id; + *id += 1; + current_id + } } } @@ -193,14 +187,14 @@ impl RpcServer for RpcServerImpl { config: DatabaseConfig, encryption_key: Option, ) -> RpcResult { - let db_id = self.get_next_db_id().await; + let db_id = self.get_next_db_id(encryption_key.is_some()).await; // Handle both Redb and Sled backends match backend { BackendType::Redb | BackendType::Sled => { - // Always create RPC databases in subdirectories to avoid conflicts with main server + // Create database directory let db_dir = if let Some(path) = &config.storage_path { - std::path::PathBuf::from(path).join(format!("rpc_db_{}", db_id)) + std::path::PathBuf::from(path) } else { std::path::PathBuf::from(&self.base_dir).join(format!("rpc_db_{}", db_id)) }; @@ -228,7 +222,13 @@ impl RpcServer for RpcServerImpl { }; // Create server instance - let server = Server::new(option).await; + let mut server = Server::new(option).await; + + // Set the selected database to the db_id for proper file naming + server.selected_db = db_id; + + // Initialize the storage to create the database file + let _ = server.current_storage(); // Store the server let mut servers = self.servers.write().await; @@ -309,9 +309,16 @@ impl RpcServer for RpcServerImpl { async fn delete_database(&self, db_id: u64) -> RpcResult { let mut servers = self.servers.write().await; - if let Some(server) = servers.remove(&db_id) { - // TODO: Clean up database files - let _ = server; + if let Some(_server) = servers.remove(&db_id) { + // Clean up database files + let db_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}.db", db_id)); + if db_path.exists() { + if db_path.is_dir() { + std::fs::remove_dir_all(&db_path).ok(); + } else { + std::fs::remove_file(&db_path).ok(); + } + } Ok(true) } else { Ok(false) diff --git a/src/rpc_server.rs b/src/rpc_server.rs index a8e9048..88ab432 100644 --- a/src/rpc_server.rs +++ b/src/rpc_server.rs @@ -3,12 +3,11 @@ use jsonrpsee::server::{ServerBuilder, ServerHandle}; use jsonrpsee::RpcModule; use crate::rpc::{RpcServer, RpcServerImpl}; -use crate::options::DBOption; /// Start the RPC server on the specified address -pub async fn start_rpc_server(addr: SocketAddr, base_dir: String, db_option: DBOption) -> Result> { +pub async fn start_rpc_server(addr: SocketAddr, base_dir: String, backend: crate::options::BackendType) -> Result> { // Create the RPC server implementation - let rpc_impl = RpcServerImpl::new(base_dir, db_option); + let rpc_impl = RpcServerImpl::new(base_dir, backend); // Create the RPC module let mut module = RpcModule::new(()); @@ -36,18 +35,9 @@ mod tests { async fn test_rpc_server_startup() { let addr = "127.0.0.1:0".parse().unwrap(); // Use port 0 for auto-assignment let base_dir = "/tmp/test_rpc".to_string(); + let backend = crate::options::BackendType::Redb; // Default for test - // Create a dummy DBOption for testing - let db_option = crate::options::DBOption { - dir: base_dir.clone(), - port: 0, - debug: false, - encryption_key: None, - encrypt: false, - backend: crate::options::BackendType::Redb, - }; - - let handle = start_rpc_server(addr, base_dir, db_option).await.unwrap(); + let handle = start_rpc_server(addr, base_dir, backend).await.unwrap(); // Give the server a moment to start tokio::time::sleep(Duration::from_millis(100)).await; -- 2.43.0 From bdf363016afc327b23b640ca845eba292ce341e6 Mon Sep 17 00:00:00 2001 From: Maxime Van Hees Date: Fri, 12 Sep 2025 17:11:50 +0200 Subject: [PATCH 3/6] WIP: adding access management control to db instances --- src/cmd.rs | 75 +++++++++++-- src/rpc.rs | 294 +++++++++++++++++++++++++++++++++++++++++++++++++- src/server.rs | 12 +++ 3 files changed, 373 insertions(+), 8 deletions(-) diff --git a/src/cmd.rs b/src/cmd.rs index 176ed2f..e3bcdda 100644 --- a/src/cmd.rs +++ b/src/cmd.rs @@ -6,7 +6,7 @@ use futures::future::select_all; pub enum Cmd { Ping, Echo(String), - Select(u64), // Changed from u16 to u64 + Select(u64, Option), // db_index, optional_key Get(String), Set(String, String), SetPx(String, String, u128), @@ -98,11 +98,18 @@ impl Cmd { Ok(( match cmd[0].to_lowercase().as_str() { "select" => { - if cmd.len() != 2 { + if cmd.len() < 2 || cmd.len() > 4 { return Err(DBError("wrong number of arguments for SELECT".to_string())); } let idx = cmd[1].parse::().map_err(|_| DBError("ERR DB index is not an integer".to_string()))?; - Cmd::Select(idx) + let key = if cmd.len() == 4 && cmd[2].to_lowercase() == "key" { + Some(cmd[3].clone()) + } else if cmd.len() == 2 { + None + } else { + return Err(DBError("ERR syntax error".to_string())); + }; + Cmd::Select(idx, key) } "echo" => Cmd::Echo(cmd[1].clone()), "ping" => Cmd::Ping, @@ -642,7 +649,7 @@ impl Cmd { } match self { - Cmd::Select(db) => select_cmd(server, db).await, + Cmd::Select(db, key) => select_cmd(server, db, key).await, Cmd::Ping => Ok(Protocol::SimpleString("PONG".to_string())), Cmd::Echo(s) => Ok(Protocol::BulkString(s)), Cmd::Get(k) => get_cmd(server, &k).await, @@ -736,7 +743,14 @@ impl Cmd { pub fn to_protocol(self) -> Protocol { match self { - Cmd::Select(db) => Protocol::Array(vec![Protocol::BulkString("select".to_string()), Protocol::BulkString(db.to_string())]), + Cmd::Select(db, key) => { + let mut arr = vec![Protocol::BulkString("select".to_string()), Protocol::BulkString(db.to_string())]; + if let Some(k) = key { + arr.push(Protocol::BulkString("key".to_string())); + arr.push(Protocol::BulkString(k)); + } + Protocol::Array(arr) + } Cmd::Ping => Protocol::Array(vec![Protocol::BulkString("ping".to_string())]), Cmd::Echo(s) => Protocol::Array(vec![Protocol::BulkString("echo".to_string()), Protocol::BulkString(s)]), Cmd::Get(k) => Protocol::Array(vec![Protocol::BulkString("get".to_string()), Protocol::BulkString(k)]), @@ -753,9 +767,44 @@ async fn flushdb_cmd(server: &mut Server) -> Result { } } -async fn select_cmd(server: &mut Server, db: u64) -> Result { - // Test if we can access the database (this will create it if needed) +async fn select_cmd(server: &mut Server, db: u64, key: Option) -> Result { + // Load database metadata + let meta = match crate::rpc::RpcServerImpl::load_meta_static(&server.option.dir, db).await { + Ok(m) => m, + Err(_) => { + // If meta doesn't exist, create default + let default_meta = crate::rpc::DatabaseMeta { + public: true, + keys: std::collections::HashMap::new(), + }; + if let Err(_) = crate::rpc::RpcServerImpl::save_meta_static(&server.option.dir, db, &default_meta).await { + return Ok(Protocol::err("ERR failed to initialize database metadata")); + } + default_meta + } + }; + + // Check access permissions + let permissions = if meta.public { + // Public database - full access + Some(crate::rpc::Permissions::ReadWrite) + } else if let Some(key_str) = key { + // Private database - check key + let hash = crate::rpc::hash_key(&key_str); + if let Some(access_key) = meta.keys.get(&hash) { + Some(access_key.permissions.clone()) + } else { + return Ok(Protocol::err("ERR invalid access key")); + } + } else { + return Ok(Protocol::err("ERR access key required for private database")); + }; + + // Set selected database and permissions server.selected_db = db; + server.current_permissions = permissions; + + // Test if we can access the database (this will create it if needed) match server.current_storage() { Ok(_) => Ok(Protocol::SimpleString("OK".to_string())), Err(e) => Ok(Protocol::err(&e.0)), @@ -1003,6 +1052,9 @@ async fn brpop_cmd(server: &Server, keys: &[String], timeout_secs: f64) -> Resul } async fn lpush_cmd(server: &Server, key: &str, elements: &[String]) -> Result { + if !server.has_write_permission() { + return Ok(Protocol::err("ERR write permission denied")); + } match server.current_storage()?.lpush(key, elements.to_vec()) { Ok(len) => { // Attempt to deliver to any blocked BLPOP waiters @@ -1134,6 +1186,9 @@ async fn type_cmd(server: &Server, k: &String) -> Result { } async fn del_cmd(server: &Server, k: &str) -> Result { + if !server.has_write_permission() { + return Ok(Protocol::err("ERR write permission denied")); + } server.current_storage()?.del(k.to_string())?; Ok(Protocol::SimpleString("1".to_string())) } @@ -1159,6 +1214,9 @@ async fn set_px_cmd( } async fn set_cmd(server: &Server, k: &str, v: &str) -> Result { + if !server.has_write_permission() { + return Ok(Protocol::err("ERR write permission denied")); + } server.current_storage()?.set(k.to_string(), v.to_string())?; Ok(Protocol::SimpleString("OK".to_string())) } @@ -1273,6 +1331,9 @@ async fn get_cmd(server: &Server, k: &str) -> Result { // Hash command implementations async fn hset_cmd(server: &Server, key: &str, pairs: &[(String, String)]) -> Result { + if !server.has_write_permission() { + return Ok(Protocol::err("ERR write permission denied")); + } let new_fields = server.current_storage()?.hset(key, pairs.to_vec())?; Ok(Protocol::SimpleString(new_fields.to_string())) } diff --git a/src/rpc.rs b/src/rpc.rs index b791bec..afbf34c 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use tokio::sync::RwLock; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use crate::server::Server; use crate::options::DBOption; @@ -39,6 +40,43 @@ pub struct DatabaseInfo { pub last_access: Option, } +/// Access permissions for database keys +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum Permissions { + Read, + ReadWrite, +} + +/// Access key information +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessKey { + pub hash: String, + pub permissions: Permissions, + pub created_at: u64, +} + +/// Database metadata containing access keys +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DatabaseMeta { + pub public: bool, + pub keys: HashMap, +} + +/// Access key information returned by RPC +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessKeyInfo { + pub hash: String, + pub permissions: Permissions, + pub created_at: u64, +} + +/// Hash a plaintext key using SHA-256 +pub fn hash_key(key: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(key.as_bytes()); + format!("{:x}", hasher.finalize()) +} + /// RPC trait for HeroDB management #[rpc(server, client, namespace = "herodb")] pub trait Rpc { @@ -70,6 +108,22 @@ pub trait Rpc { /// Get server statistics #[method(name = "getServerStats")] async fn get_server_stats(&self) -> RpcResult>; + + /// Add an access key to a database + #[method(name = "addAccessKey")] + async fn add_access_key(&self, db_id: u64, key: String, permissions: String) -> RpcResult; + + /// Delete an access key from a database + #[method(name = "deleteAccessKey")] + async fn delete_access_key(&self, db_id: u64, key_hash: String) -> RpcResult; + + /// List all access keys for a database + #[method(name = "listAccessKeys")] + async fn list_access_keys(&self, db_id: u64) -> RpcResult>; + + /// Set database public/private status + #[method(name = "setDatabasePublic")] + async fn set_database_public(&self, db_id: u64, public: bool) -> RpcResult; } /// RPC Server implementation @@ -84,6 +138,8 @@ pub struct RpcServerImpl { next_encrypted_id: Arc>, /// Default backend type backend: crate::options::BackendType, + /// Encryption keys for databases + encryption_keys: Arc>>>, } impl RpcServerImpl { @@ -95,6 +151,7 @@ impl RpcServerImpl { next_unencrypted_id: Arc::new(RwLock::new(0)), next_encrypted_id: Arc::new(RwLock::new(10)), backend, + encryption_keys: Arc::new(RwLock::new(HashMap::new())), } } @@ -177,6 +234,166 @@ impl RpcServerImpl { current_id } } + + /// Load database metadata from file (static version) + pub async fn load_meta_static(base_dir: &str, db_id: u64) -> Result { + let meta_path = std::path::PathBuf::from(base_dir).join(format!("{}_meta.json", db_id)); + + // If meta file doesn't exist, return default + if !meta_path.exists() { + return Ok(DatabaseMeta { + public: true, + keys: HashMap::new(), + }); + } + + // Read file + let content = std::fs::read(&meta_path) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to read meta file: {}", e), + None::<()> + ))?; + + let json_str = String::from_utf8(content) + .map_err(|_| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + "Invalid UTF-8 in meta file", + None::<()> + ))?; + + serde_json::from_str(&json_str) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to parse meta JSON: {}", e), + None::<()> + )) + } + + /// Load database metadata from file + async fn load_meta(&self, db_id: u64) -> Result { + let meta_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}_meta.json", db_id)); + + // If meta file doesn't exist, create default + if !meta_path.exists() { + let default_meta = DatabaseMeta { + public: true, + keys: HashMap::new(), + }; + self.save_meta(db_id, &default_meta).await?; + return Ok(default_meta); + } + + // Read and potentially decrypt + let content = std::fs::read(&meta_path) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to read meta file: {}", e), + None::<()> + ))?; + + let json_str = if db_id >= 10 { + // Encrypted database, decrypt meta + if let Some(key) = self.encryption_keys.read().await.get(&db_id).and_then(|k| k.as_ref()) { + use crate::crypto::CryptoFactory; + let crypto = CryptoFactory::new(key.as_bytes()); + String::from_utf8(crypto.decrypt(&content) + .map_err(|_| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + "Failed to decrypt meta file", + None::<()> + ))?) + .map_err(|_| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + "Invalid UTF-8 in decrypted meta", + None::<()> + ))? + } else { + return Err(jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + "Encryption key not found for encrypted database", + None::<()> + )); + } + } else { + String::from_utf8(content) + .map_err(|_| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + "Invalid UTF-8 in meta file", + None::<()> + ))? + }; + + serde_json::from_str(&json_str) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to parse meta JSON: {}", e), + None::<()> + )) + } + + /// Save database metadata to file (static version) + pub async fn save_meta_static(base_dir: &str, db_id: u64, meta: &DatabaseMeta) -> Result<(), jsonrpsee::types::ErrorObjectOwned> { + let meta_path = std::path::PathBuf::from(base_dir).join(format!("{}_meta.json", db_id)); + + let json_str = serde_json::to_string(meta) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to serialize meta: {}", e), + None::<()> + ))?; + + std::fs::write(&meta_path, json_str) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to write meta file: {}", e), + None::<()> + ))?; + + Ok(()) + } + + /// Save database metadata to file + async fn save_meta(&self, db_id: u64, meta: &DatabaseMeta) -> Result<(), jsonrpsee::types::ErrorObjectOwned> { + let meta_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}_meta.json", db_id)); + + let json_str = serde_json::to_string(meta) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to serialize meta: {}", e), + None::<()> + ))?; + + if db_id >= 10 { + // Encrypted database, encrypt meta + if let Some(key) = self.encryption_keys.read().await.get(&db_id).and_then(|k| k.as_ref()) { + use crate::crypto::CryptoFactory; + let crypto = CryptoFactory::new(key.as_bytes()); + let encrypted = crypto.encrypt(json_str.as_bytes()); + std::fs::write(&meta_path, encrypted) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to write encrypted meta file: {}", e), + None::<()> + ))?; + } else { + return Err(jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + "Encryption key not found for encrypted database", + None::<()> + )); + } + } else { + std::fs::write(&meta_path, json_str) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to write meta file: {}", e), + None::<()> + ))?; + } + + Ok(()) + } } #[jsonrpsee::core::async_trait] @@ -213,7 +430,7 @@ impl RpcServer for RpcServerImpl { dir: db_dir.to_string_lossy().to_string(), port: 0, // Not used for RPC-managed databases debug: false, - encryption_key, + encryption_key: encryption_key.clone(), encrypt, backend: match backend { BackendType::Redb => crate::options::BackendType::Redb, @@ -230,6 +447,19 @@ impl RpcServer for RpcServerImpl { // Initialize the storage to create the database file let _ = server.current_storage(); + // Store the encryption key + { + let mut keys = self.encryption_keys.write().await; + keys.insert(db_id, encryption_key.clone()); + } + + // Initialize meta file + let meta = DatabaseMeta { + public: true, + keys: HashMap::new(), + }; + self.save_meta(db_id, &meta).await?; + // Store the server let mut servers = self.servers.write().await; servers.insert(db_id, Arc::new(server)); @@ -339,4 +569,66 @@ impl RpcServer for RpcServerImpl { Ok(stats) } + + async fn add_access_key(&self, db_id: u64, key: String, permissions: String) -> RpcResult { + let mut meta = self.load_meta(db_id).await?; + + let perms = match permissions.to_lowercase().as_str() { + "read" => Permissions::Read, + "readwrite" => Permissions::ReadWrite, + _ => return Err(jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + "Invalid permissions: use 'read' or 'readwrite'", + None::<()> + )), + }; + + let hash = hash_key(&key); + let access_key = AccessKey { + hash: hash.clone(), + permissions: perms, + created_at: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), + }; + + meta.keys.insert(hash, access_key); + self.save_meta(db_id, &meta).await?; + Ok(true) + } + + async fn delete_access_key(&self, db_id: u64, key_hash: String) -> RpcResult { + let mut meta = self.load_meta(db_id).await?; + + if meta.keys.remove(&key_hash).is_some() { + // If no keys left, make database public + if meta.keys.is_empty() { + meta.public = true; + } + self.save_meta(db_id, &meta).await?; + Ok(true) + } else { + Ok(false) + } + } + + async fn list_access_keys(&self, db_id: u64) -> RpcResult> { + let meta = self.load_meta(db_id).await?; + let keys: Vec = meta.keys.values() + .map(|k| AccessKeyInfo { + hash: k.hash.clone(), + permissions: k.permissions.clone(), + created_at: k.created_at, + }) + .collect(); + Ok(keys) + } + + async fn set_database_public(&self, db_id: u64, public: bool) -> RpcResult { + let mut meta = self.load_meta(db_id).await?; + meta.public = public; + self.save_meta(db_id, &meta).await?; + Ok(true) + } } \ No newline at end of file diff --git a/src/server.rs b/src/server.rs index a6e43e2..63864c6 100644 --- a/src/server.rs +++ b/src/server.rs @@ -22,6 +22,7 @@ pub struct Server { pub client_name: Option, pub selected_db: u64, // Changed from usize to u64 pub queued_cmd: Option>, + pub current_permissions: Option, // BLPOP waiter registry: per (db_index, key) FIFO of waiters pub list_waiters: Arc>>>>, @@ -48,6 +49,7 @@ impl Server { client_name: None, selected_db: 0, queued_cmd: None, + current_permissions: None, list_waiters: Arc::new(Mutex::new(HashMap::new())), waiter_seq: Arc::new(AtomicU64::new(1)), @@ -101,6 +103,16 @@ impl Server { self.option.encrypt && db_index >= 10 } + /// Check if current permissions allow read operations + pub fn has_read_permission(&self) -> bool { + matches!(self.current_permissions, Some(crate::rpc::Permissions::Read) | Some(crate::rpc::Permissions::ReadWrite)) + } + + /// Check if current permissions allow write operations + pub fn has_write_permission(&self) -> bool { + matches!(self.current_permissions, Some(crate::rpc::Permissions::ReadWrite)) + } + // ----- BLPOP waiter helpers ----- pub async fn register_waiter(&self, db_index: u64, key: &str, side: PopSide) -> (u64, oneshot::Receiver<(String, String)>) { -- 2.43.0 From da325a9659bf77a9cdcea4832b5780f43355b41a Mon Sep 17 00:00:00 2001 From: Maxime Van Hees Date: Mon, 15 Sep 2025 10:34:03 +0200 Subject: [PATCH 4/6] fix bug where meta files where not auto-created upon starting + fix bug where meta json files were actually binary + improved access control to database instances --- src/main.rs | 5 ++- src/rpc.rs | 96 +++++++++++------------------------------------------ 2 files changed, 24 insertions(+), 77 deletions(-) diff --git a/src/main.rs b/src/main.rs index 6b88c33..f98876f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -76,9 +76,12 @@ async fn main() { // new server let mut server = server::Server::new(option).await; - // Initialize the default database storage + // Initialize the default database storage (creates 0.db) let _ = server.current_storage(); + // Ensure default meta for DB 0 exists (public by default if missing) + let _ = herodb::rpc::RpcServerImpl::load_meta_static(&server.option.dir, 0).await; + // Add a small delay to ensure the port is ready tokio::time::sleep(std::time::Duration::from_millis(100)).await; diff --git a/src/rpc.rs b/src/rpc.rs index afbf34c..ccca52b 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -239,29 +239,25 @@ impl RpcServerImpl { pub async fn load_meta_static(base_dir: &str, db_id: u64) -> Result { let meta_path = std::path::PathBuf::from(base_dir).join(format!("{}_meta.json", db_id)); - // If meta file doesn't exist, return default + // If meta file doesn't exist, create and persist default if !meta_path.exists() { - return Ok(DatabaseMeta { + let default_meta = DatabaseMeta { public: true, keys: HashMap::new(), - }); + }; + // Persist default metadata to disk + Self::save_meta_static(base_dir, db_id, &default_meta).await?; + return Ok(default_meta); } - // Read file - let content = std::fs::read(&meta_path) + // Read file as UTF-8 JSON + let json_str = std::fs::read_to_string(&meta_path) .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( -32000, format!("Failed to read meta file: {}", e), None::<()> ))?; - let json_str = String::from_utf8(content) - .map_err(|_| jsonrpsee::types::ErrorObjectOwned::owned( - -32000, - "Invalid UTF-8 in meta file", - None::<()> - ))?; - serde_json::from_str(&json_str) .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( -32000, @@ -274,7 +270,7 @@ impl RpcServerImpl { async fn load_meta(&self, db_id: u64) -> Result { let meta_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}_meta.json", db_id)); - // If meta file doesn't exist, create default + // If meta file doesn't exist, create and persist default if !meta_path.exists() { let default_meta = DatabaseMeta { public: true, @@ -284,46 +280,14 @@ impl RpcServerImpl { return Ok(default_meta); } - // Read and potentially decrypt - let content = std::fs::read(&meta_path) + // Read file as UTF-8 JSON (meta files are always plain JSON) + let json_str = std::fs::read_to_string(&meta_path) .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( -32000, format!("Failed to read meta file: {}", e), None::<()> ))?; - let json_str = if db_id >= 10 { - // Encrypted database, decrypt meta - if let Some(key) = self.encryption_keys.read().await.get(&db_id).and_then(|k| k.as_ref()) { - use crate::crypto::CryptoFactory; - let crypto = CryptoFactory::new(key.as_bytes()); - String::from_utf8(crypto.decrypt(&content) - .map_err(|_| jsonrpsee::types::ErrorObjectOwned::owned( - -32000, - "Failed to decrypt meta file", - None::<()> - ))?) - .map_err(|_| jsonrpsee::types::ErrorObjectOwned::owned( - -32000, - "Invalid UTF-8 in decrypted meta", - None::<()> - ))? - } else { - return Err(jsonrpsee::types::ErrorObjectOwned::owned( - -32000, - "Encryption key not found for encrypted database", - None::<()> - )); - } - } else { - String::from_utf8(content) - .map_err(|_| jsonrpsee::types::ErrorObjectOwned::owned( - -32000, - "Invalid UTF-8 in meta file", - None::<()> - ))? - }; - serde_json::from_str(&json_str) .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( -32000, @@ -336,7 +300,7 @@ impl RpcServerImpl { pub async fn save_meta_static(base_dir: &str, db_id: u64, meta: &DatabaseMeta) -> Result<(), jsonrpsee::types::ErrorObjectOwned> { let meta_path = std::path::PathBuf::from(base_dir).join(format!("{}_meta.json", db_id)); - let json_str = serde_json::to_string(meta) + let json_str = serde_json::to_string_pretty(meta) .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( -32000, format!("Failed to serialize meta: {}", e), @@ -357,40 +321,20 @@ impl RpcServerImpl { async fn save_meta(&self, db_id: u64, meta: &DatabaseMeta) -> Result<(), jsonrpsee::types::ErrorObjectOwned> { let meta_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}_meta.json", db_id)); - let json_str = serde_json::to_string(meta) + let json_str = serde_json::to_string_pretty(meta) .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( -32000, format!("Failed to serialize meta: {}", e), None::<()> ))?; - if db_id >= 10 { - // Encrypted database, encrypt meta - if let Some(key) = self.encryption_keys.read().await.get(&db_id).and_then(|k| k.as_ref()) { - use crate::crypto::CryptoFactory; - let crypto = CryptoFactory::new(key.as_bytes()); - let encrypted = crypto.encrypt(json_str.as_bytes()); - std::fs::write(&meta_path, encrypted) - .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( - -32000, - format!("Failed to write encrypted meta file: {}", e), - None::<()> - ))?; - } else { - return Err(jsonrpsee::types::ErrorObjectOwned::owned( - -32000, - "Encryption key not found for encrypted database", - None::<()> - )); - } - } else { - std::fs::write(&meta_path, json_str) - .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( - -32000, - format!("Failed to write meta file: {}", e), - None::<()> - ))?; - } + // Meta files are always stored as plain JSON (even when data DB is encrypted) + std::fs::write(&meta_path, json_str) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( + -32000, + format!("Failed to write meta file: {}", e), + None::<()> + ))?; Ok(()) } -- 2.43.0 From 1b15806a8523d610a54427a7f73fc793501b4250 Mon Sep 17 00:00:00 2001 From: Maxime Van Hees Date: Mon, 15 Sep 2025 13:45:37 +0200 Subject: [PATCH 5/6] fix invalid values in RPC response about database instance details --- src/rpc.rs | 133 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 91 insertions(+), 42 deletions(-) diff --git a/src/rpc.rs b/src/rpc.rs index ccca52b..c64d2b4 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -338,6 +338,92 @@ impl RpcServerImpl { Ok(()) } + + /// Build database file path for given server/db_id + fn db_file_path(&self, server: &Server, db_id: u64) -> std::path::PathBuf { + std::path::PathBuf::from(&server.option.dir).join(format!("{}.db", db_id)) + } + + /// Recursively compute size on disk for the database path + fn compute_size_on_disk(&self, path: &std::path::Path) -> Option { + fn dir_size(p: &std::path::Path) -> u64 { + if p.is_file() { + std::fs::metadata(p).map(|m| m.len()).unwrap_or(0) + } else if p.is_dir() { + let mut total = 0u64; + if let Ok(read) = std::fs::read_dir(p) { + for entry in read.flatten() { + total += dir_size(&entry.path()); + } + } + total + } else { + 0 + } + } + Some(dir_size(path)) + } + + /// Extract created and last access times (secs) from a path, with fallbacks + fn get_file_times_secs(path: &std::path::Path) -> (u64, Option) { + let now = std::time::SystemTime::now(); + let created = std::fs::metadata(path) + .and_then(|m| m.created().or_else(|_| m.modified())) + .unwrap_or(now) + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + let last_access = std::fs::metadata(path) + .and_then(|m| m.accessed()) + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok().map(|d| d.as_secs())); + + (created, last_access) + } + + /// Compose a DatabaseInfo by probing storage, metadata and filesystem + async fn build_database_info(&self, db_id: u64, server: &Server) -> DatabaseInfo { + // Probe storage to determine encryption state + let storage = server.current_storage().ok(); + let encrypted = storage.as_ref().map(|s| s.is_encrypted()).unwrap_or(server.option.encrypt); + + // Load meta to get access key count + let meta = Self::load_meta_static(&self.base_dir, db_id).await.unwrap_or(DatabaseMeta { + public: true, + keys: HashMap::new(), + }); + let key_count = Some(meta.keys.len() as u64); + + // Compute size on disk and timestamps + let db_path = self.db_file_path(server, db_id); + let size_on_disk = self.compute_size_on_disk(&db_path); + + let meta_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}_meta.json", db_id)); + let (created_at, last_access) = if meta_path.exists() { + Self::get_file_times_secs(&meta_path) + } else { + Self::get_file_times_secs(&db_path) + }; + + let backend = match server.option.backend { + crate::options::BackendType::Redb => BackendType::Redb, + crate::options::BackendType::Sled => BackendType::Sled, + }; + + DatabaseInfo { + id: db_id, + name: None, + backend, + encrypted, + redis_version: Some("7.0".to_string()), + storage_path: Some(server.option.dir.clone()), + size_on_disk, + key_count, + created_at, + last_access, + } + } } #[jsonrpsee::core::async_trait] @@ -426,28 +512,9 @@ impl RpcServer for RpcServerImpl { let mut result = Vec::new(); for db_id in db_ids { - // Try to get or create server for this database if let Ok(server) = self.get_or_create_server(db_id).await { - let backend = match server.option.backend { - crate::options::BackendType::Redb => BackendType::Redb, - crate::options::BackendType::Sled => BackendType::Sled, - }; - - let info = DatabaseInfo { - id: db_id, - name: None, // TODO: Store name in server metadata - backend, - encrypted: server.option.encrypt, - redis_version: Some("7.0".to_string()), // Default Redis compatibility - storage_path: Some(server.option.dir.clone()), - size_on_disk: None, // TODO: Calculate actual size - key_count: None, // TODO: Get key count from storage - created_at: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(), - last_access: None, - }; + // Build accurate info from storage/meta/fs + let info = self.build_database_info(db_id, &server).await; result.push(info); } } @@ -457,27 +524,9 @@ impl RpcServer for RpcServerImpl { async fn get_database_info(&self, db_id: u64) -> RpcResult { let server = self.get_or_create_server(db_id).await?; - - let backend = match server.option.backend { - crate::options::BackendType::Redb => BackendType::Redb, - crate::options::BackendType::Sled => BackendType::Sled, - }; - - Ok(DatabaseInfo { - id: db_id, - name: None, - backend, - encrypted: server.option.encrypt, - redis_version: Some("7.0".to_string()), - storage_path: Some(server.option.dir.clone()), - size_on_disk: None, - key_count: None, - created_at: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(), - last_access: None, - }) + // Build accurate info from storage/meta/fs + let info = self.build_database_info(db_id, &server).await; + Ok(info) } async fn delete_database(&self, db_id: u64) -> RpcResult { -- 2.43.0 From b8ca73397d79fdc4b4cbc305fb384480bdc707b6 Mon Sep 17 00:00:00 2001 From: Maxime Van Hees Date: Tue, 16 Sep 2025 16:06:47 +0200 Subject: [PATCH 6/6] implemented 0.db as admin database architecture + updated test file --- src/admin_meta.rs | 361 +++++++++++++++++++++++++++++++ src/cmd.rs | 79 ++++--- src/lib.rs | 1 + src/main.rs | 36 ++- src/options.rs | 4 + src/rpc.rs | 195 ++++++----------- src/rpc_server.rs | 6 +- src/server.rs | 58 ++--- tests/debug_hset.rs | 7 + tests/debug_hset_simple.rs | 10 +- tests/redis_integration_tests.rs | 12 +- tests/redis_tests.rs | 15 +- tests/simple_integration_test.rs | 21 +- tests/simple_redis_test.rs | 30 ++- tests/usage_suite.rs | 19 +- 15 files changed, 631 insertions(+), 223 deletions(-) create mode 100644 src/admin_meta.rs diff --git a/src/admin_meta.rs b/src/admin_meta.rs new file mode 100644 index 0000000..78ba22a --- /dev/null +++ b/src/admin_meta.rs @@ -0,0 +1,361 @@ +use std::path::PathBuf; +use std::sync::{Arc, OnceLock, Mutex, RwLock}; +use std::collections::HashMap; + +use crate::error::DBError; +use crate::options; +use crate::rpc::Permissions; +use crate::storage::Storage; +use crate::storage_sled::SledStorage; +use crate::storage_trait::StorageBackend; + +// Key builders +fn k_admin_next_id() -> &'static str { + "admin:next_id" +} +fn k_admin_dbs() -> &'static str { + "admin:dbs" +} +fn k_meta_db(id: u64) -> String { + format!("meta:db:{}", id) +} +fn k_meta_db_keys(id: u64) -> String { + format!("meta:db:{}:keys", id) +} +fn k_meta_db_enc(id: u64) -> String { + format!("meta:db:{}:enc", id) +} + +// Global cache of admin DB 0 handles per base_dir to avoid sled/reDB file-lock contention +// and to correctly isolate different test instances with distinct directories. +static ADMIN_STORAGES: OnceLock>>> = OnceLock::new(); + +// Global registry for data DB storages to avoid double-open across process. +static DATA_STORAGES: OnceLock>>> = OnceLock::new(); +static DATA_INIT_LOCK: Mutex<()> = Mutex::new(()); + +fn init_admin_storage( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, +) -> Result, DBError> { + let db_file = PathBuf::from(base_dir).join("0.db"); + if let Some(parent_dir) = db_file.parent() { + std::fs::create_dir_all(parent_dir).map_err(|e| { + DBError(format!("Failed to create directory {}: {}", parent_dir.display(), e)) + })?; + } + let storage: Arc = match backend { + options::BackendType::Redb => Arc::new(Storage::new(&db_file, true, Some(admin_secret))?), + options::BackendType::Sled => Arc::new(SledStorage::new(&db_file, true, Some(admin_secret))?), + }; + Ok(storage) +} + +// Get or initialize a cached handle to admin DB 0 per base_dir (thread-safe, no double-open race) +pub fn open_admin_storage( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, +) -> Result, DBError> { + let map = ADMIN_STORAGES.get_or_init(|| RwLock::new(HashMap::new())); + // Fast path + if let Some(st) = map.read().unwrap().get(base_dir) { + return Ok(st.clone()); + } + // Slow path with write lock + { + let mut w = map.write().unwrap(); + if let Some(st) = w.get(base_dir) { + return Ok(st.clone()); + } + let st = init_admin_storage(base_dir, backend, admin_secret)?; + w.insert(base_dir.to_string(), st.clone()); + return Ok(st); + } +} + +// Ensure admin structures exist in encrypted DB 0 +pub fn ensure_bootstrap( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, +) -> Result<(), DBError> { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + + // Initialize next id if missing + if !admin.exists(k_admin_next_id())? { + admin.set(k_admin_next_id().to_string(), "1".to_string())?; + } + // admin:dbs is a hash; it's fine if it doesn't exist (hlen -> 0) + Ok(()) +} + +// Get or initialize a shared handle to a data DB (> 0), avoiding double-open across subsystems +pub fn open_data_storage( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, + id: u64, +) -> Result, DBError> { + if id == 0 { + return open_admin_storage(base_dir, backend, admin_secret); + } + + // Validate existence in admin metadata + if !db_exists(base_dir, backend.clone(), admin_secret, id)? { + return Err(DBError(format!( + "Cannot open database instance {}, as that database instance does not exist.", + id + ))); + } + + let map = DATA_STORAGES.get_or_init(|| RwLock::new(HashMap::new())); + // Fast path + if let Some(st) = map.read().unwrap().get(&id) { + return Ok(st.clone()); + } + + // Slow path with init lock + let _guard = DATA_INIT_LOCK.lock().unwrap(); + if let Some(st) = map.read().unwrap().get(&id) { + return Ok(st.clone()); + } + + // Determine per-db encryption + let enc = get_enc_key(base_dir, backend.clone(), admin_secret, id)?; + let should_encrypt = enc.is_some(); + + // Build database file path and ensure parent dir exists + let db_file = PathBuf::from(base_dir).join(format!("{}.db", id)); + if let Some(parent_dir) = db_file.parent() { + std::fs::create_dir_all(parent_dir).map_err(|e| { + DBError(format!("Failed to create directory {}: {}", parent_dir.display(), e)) + })?; + } + + // Open storage + let storage: Arc = match backend { + options::BackendType::Redb => Arc::new(Storage::new(&db_file, should_encrypt, enc.as_deref())?), + options::BackendType::Sled => Arc::new(SledStorage::new(&db_file, should_encrypt, enc.as_deref())?), + }; + + // Publish to registry + map.write().unwrap().insert(id, storage.clone()); + Ok(storage) +} + +// Allocate the next DB id and persist new pointer +pub fn allocate_next_id( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, +) -> Result { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + let cur = admin + .get(k_admin_next_id())? + .unwrap_or_else(|| "1".to_string()); + let id: u64 = cur.parse().unwrap_or(1); + let next = id.checked_add(1).ok_or_else(|| DBError("next_id overflow".into()))?; + admin.set(k_admin_next_id().to_string(), next.to_string())?; + + // Register into admin:dbs set/hash + let _ = admin.hset(k_admin_dbs(), vec![(id.to_string(), "1".to_string())])?; + + // Default meta for the new db: public true + let meta_key = k_meta_db(id); + let _ = admin.hset(&meta_key, vec![("public".to_string(), "true".to_string())])?; + + Ok(id) +} + +// Check existence of a db id in admin:dbs +pub fn db_exists( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, + id: u64, +) -> Result { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + Ok(admin.hexists(k_admin_dbs(), &id.to_string())?) +} + +// Get per-db encryption key, if any +pub fn get_enc_key( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, + id: u64, +) -> Result, DBError> { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + admin.get(&k_meta_db_enc(id)) +} + +// Set per-db encryption key (called during create) +pub fn set_enc_key( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, + id: u64, + key: &str, +) -> Result<(), DBError> { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + admin.set(k_meta_db_enc(id), key.to_string()) +} + +// Set database public flag +pub fn set_database_public( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, + id: u64, + public: bool, +) -> Result<(), DBError> { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + let mk = k_meta_db(id); + let _ = admin.hset(&mk, vec![("public".to_string(), public.to_string())])?; + Ok(()) +} + +// Internal: load public flag; default to true when meta missing +fn load_public( + admin: &Arc, + id: u64, +) -> Result { + let mk = k_meta_db(id); + match admin.hget(&mk, "public")? { + Some(v) => Ok(v == "true"), + None => Ok(true), + } +} + +// Add access key for db (value format: "Read:ts" or "ReadWrite:ts") +pub fn add_access_key( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, + id: u64, + key_plain: &str, + perms: Permissions, +) -> Result<(), DBError> { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + let hash = crate::rpc::hash_key(key_plain); + let v = match perms { + Permissions::Read => format!("Read:{}", now_secs()), + Permissions::ReadWrite => format!("ReadWrite:{}", now_secs()), + }; + let _ = admin.hset(&k_meta_db_keys(id), vec![(hash, v)])?; + Ok(()) +} + +// Delete access key by hash +pub fn delete_access_key( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, + id: u64, + key_hash: &str, +) -> Result { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + let n = admin.hdel(&k_meta_db_keys(id), vec![key_hash.to_string()])?; + Ok(n > 0) +} + +// List access keys, returning (hash, perms, created_at_secs) +pub fn list_access_keys( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, + id: u64, +) -> Result, DBError> { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + let pairs = admin.hgetall(&k_meta_db_keys(id))?; + let mut out = Vec::new(); + for (hash, val) in pairs { + let (perm, ts) = parse_perm_value(&val); + out.push((hash, perm, ts)); + } + Ok(out) +} + +// Verify access permission for db id with optional key +// Returns: +// - Ok(Some(Permissions)) when access is allowed +// - Ok(None) when not allowed or db missing (caller can distinguish by calling db_exists) +pub fn verify_access( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, + id: u64, + key_opt: Option<&str>, +) -> Result, DBError> { + // Admin DB 0: require exact admin_secret + if id == 0 { + if let Some(k) = key_opt { + if k == admin_secret { + return Ok(Some(Permissions::ReadWrite)); + } + } + return Ok(None); + } + + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + if !admin.hexists(k_admin_dbs(), &id.to_string())? { + return Ok(None); + } + + // Public? + if load_public(&admin, id)? { + return Ok(Some(Permissions::ReadWrite)); + } + + // Private: require key and verify + if let Some(k) = key_opt { + let hash = crate::rpc::hash_key(k); + if let Some(v) = admin.hget(&k_meta_db_keys(id), &hash)? { + let (perm, _ts) = parse_perm_value(&v); + return Ok(Some(perm)); + } + } + Ok(None) +} + +// Enumerate all db ids +pub fn list_dbs( + base_dir: &str, + backend: options::BackendType, + admin_secret: &str, +) -> Result, DBError> { + let admin = open_admin_storage(base_dir, backend, admin_secret)?; + let ids = admin.hkeys(k_admin_dbs())?; + let mut out = Vec::new(); + for s in ids { + if let Ok(v) = s.parse() { + out.push(v); + } + } + Ok(out) +} + +// Helper: parse permission value "Read:ts" or "ReadWrite:ts" +fn parse_perm_value(v: &str) -> (Permissions, u64) { + let mut parts = v.split(':'); + let p = parts.next().unwrap_or("Read"); + let ts = parts + .next() + .and_then(|s| s.parse().ok()) + .unwrap_or(0u64); + let perm = match p { + "ReadWrite" => Permissions::ReadWrite, + _ => Permissions::Read, + }; + (perm, ts) +} + +fn now_secs() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} \ No newline at end of file diff --git a/src/cmd.rs b/src/cmd.rs index e3bcdda..98cbacd 100644 --- a/src/cmd.rs +++ b/src/cmd.rs @@ -768,43 +768,64 @@ async fn flushdb_cmd(server: &mut Server) -> Result { } async fn select_cmd(server: &mut Server, db: u64, key: Option) -> Result { - // Load database metadata - let meta = match crate::rpc::RpcServerImpl::load_meta_static(&server.option.dir, db).await { - Ok(m) => m, - Err(_) => { - // If meta doesn't exist, create default - let default_meta = crate::rpc::DatabaseMeta { - public: true, - keys: std::collections::HashMap::new(), - }; - if let Err(_) = crate::rpc::RpcServerImpl::save_meta_static(&server.option.dir, db, &default_meta).await { - return Ok(Protocol::err("ERR failed to initialize database metadata")); + // Authorization and existence checks via admin DB 0 + // DB 0: require KEY admin-secret + if db == 0 { + match key { + Some(k) if k == server.option.admin_secret => { + server.selected_db = 0; + server.current_permissions = Some(crate::rpc::Permissions::ReadWrite); + // Will create encrypted 0.db if missing + match server.current_storage() { + Ok(_) => return Ok(Protocol::SimpleString("OK".to_string())), + Err(e) => return Ok(Protocol::err(&e.0)), + } + } + _ => { + return Ok(Protocol::err("ERR invalid access key")); } - default_meta } + } + + // DB > 0: must exist in admin:dbs + let exists = match crate::admin_meta::db_exists( + &server.option.dir, + server.option.backend.clone(), + &server.option.admin_secret, + db, + ) { + Ok(b) => b, + Err(e) => return Ok(Protocol::err(&e.0)), }; - // Check access permissions - let permissions = if meta.public { - // Public database - full access - Some(crate::rpc::Permissions::ReadWrite) - } else if let Some(key_str) = key { - // Private database - check key - let hash = crate::rpc::hash_key(&key_str); - if let Some(access_key) = meta.keys.get(&hash) { - Some(access_key.permissions.clone()) - } else { - return Ok(Protocol::err("ERR invalid access key")); - } - } else { - return Ok(Protocol::err("ERR access key required for private database")); + if !exists { + return Ok(Protocol::err(&format!( + "Cannot open database instance {}, as that database instance does not exist.", + db + ))); + } + + // Verify permissions (public => RW; private => use key) + let perms_opt = match crate::admin_meta::verify_access( + &server.option.dir, + server.option.backend.clone(), + &server.option.admin_secret, + db, + key.as_deref(), + ) { + Ok(p) => p, + Err(e) => return Ok(Protocol::err(&e.0)), }; - // Set selected database and permissions + let perms = match perms_opt { + Some(p) => p, + None => return Ok(Protocol::err("ERR invalid access key")), + }; + + // Set selected database and permissions, then open storage server.selected_db = db; - server.current_permissions = permissions; + server.current_permissions = Some(perms); - // Test if we can access the database (this will create it if needed) match server.current_storage() { Ok(_) => Ok(Protocol::SimpleString("OK".to_string())), Err(e) => Ok(Protocol::err(&e.0)), diff --git a/src/lib.rs b/src/lib.rs index 85dd4e4..66a2990 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,3 +10,4 @@ pub mod server; pub mod storage; pub mod storage_trait; // Add this pub mod storage_sled; // Add this +pub mod admin_meta; diff --git a/src/main.rs b/src/main.rs index f98876f..233e675 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,12 +23,11 @@ struct Args { #[arg(long)] debug: bool, - - /// Master encryption key for encrypted databases + /// Master encryption key for encrypted databases (deprecated; ignored for data DBs) #[arg(long)] encryption_key: Option, - /// Encrypt the database + /// Encrypt the database (deprecated; ignored for data DBs) #[arg(long)] encrypt: bool, @@ -43,6 +42,10 @@ struct Args { /// Use the sled backend #[arg(long)] sled: bool, + + /// Admin secret used to encrypt DB 0 and authorize admin access (required) + #[arg(long)] + admin_secret: String, } #[tokio::main] @@ -57,6 +60,16 @@ async fn main() { .await .unwrap(); + // deprecation warnings for legacy flags + if args.encrypt || args.encryption_key.is_some() { + eprintln!("warning: --encrypt and --encryption-key are deprecated and ignored for data DBs. Admin DB 0 is always encrypted with --admin-secret."); + } + // basic validation for admin secret + if args.admin_secret.trim().is_empty() { + eprintln!("error: --admin-secret must not be empty"); + std::process::exit(2); + } + // new DB option let option = herodb::options::DBOption { dir: args.dir.clone(), @@ -69,18 +82,19 @@ async fn main() { } else { herodb::options::BackendType::Redb }, + admin_secret: args.admin_secret.clone(), }; let backend = option.backend.clone(); + // Bootstrap admin DB 0 before opening any server storage + if let Err(e) = herodb::admin_meta::ensure_bootstrap(&args.dir, backend.clone(), &args.admin_secret) { + eprintln!("Failed to bootstrap admin DB 0: {}", e.0); + std::process::exit(2); + } + // new server - let mut server = server::Server::new(option).await; - - // Initialize the default database storage (creates 0.db) - let _ = server.current_storage(); - - // Ensure default meta for DB 0 exists (public by default if missing) - let _ = herodb::rpc::RpcServerImpl::load_meta_static(&server.option.dir, 0).await; + let server = server::Server::new(option).await; // Add a small delay to ensure the port is ready tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -90,7 +104,7 @@ async fn main() { let rpc_addr = format!("127.0.0.1:{}", args.rpc_port).parse().unwrap(); let base_dir = args.dir.clone(); - match rpc_server::start_rpc_server(rpc_addr, base_dir, backend).await { + match rpc_server::start_rpc_server(rpc_addr, base_dir, backend, args.admin_secret.clone()).await { Ok(handle) => { println!("RPC management server started on port {}", args.rpc_port); Some(handle) diff --git a/src/options.rs b/src/options.rs index 067183d..2ee44c2 100644 --- a/src/options.rs +++ b/src/options.rs @@ -9,7 +9,11 @@ pub struct DBOption { pub dir: String, pub port: u16, pub debug: bool, + // Deprecated for data DBs; retained for backward-compat on CLI parsing pub encrypt: bool, + // Deprecated for data DBs; retained for backward-compat on CLI parsing pub encryption_key: Option, pub backend: BackendType, + // New: required admin secret, used to encrypt DB 0 and authorize admin operations + pub admin_secret: String, } diff --git a/src/rpc.rs b/src/rpc.rs index c64d2b4..a6d4400 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -7,6 +7,7 @@ use sha2::{Digest, Sha256}; use crate::server::Server; use crate::options::DBOption; +use crate::admin_meta; /// Database backend types #[derive(Debug, Clone, Serialize, Deserialize)] @@ -140,11 +141,13 @@ pub struct RpcServerImpl { backend: crate::options::BackendType, /// Encryption keys for databases encryption_keys: Arc>>>, + /// Admin secret used to encrypt DB 0 and authorize admin access + admin_secret: String, } impl RpcServerImpl { /// Create a new RPC server instance - pub fn new(base_dir: String, backend: crate::options::BackendType) -> Self { + pub fn new(base_dir: String, backend: crate::options::BackendType, admin_secret: String) -> Self { Self { base_dir, servers: Arc::new(RwLock::new(HashMap::new())), @@ -152,6 +155,7 @@ impl RpcServerImpl { next_encrypted_id: Arc::new(RwLock::new(10)), backend, encryption_keys: Arc::new(RwLock::new(HashMap::new())), + admin_secret, } } @@ -165,9 +169,10 @@ impl RpcServerImpl { } } - // Check if database file exists - let db_path = std::path::PathBuf::from(&self.base_dir).join(format!("{}.db", db_id)); - if !db_path.exists() { + // Validate existence via admin DB 0 (metadata), not filesystem presence + let exists = admin_meta::db_exists(&self.base_dir, self.backend.clone(), &self.admin_secret, db_id) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?; + if !exists { return Err(jsonrpsee::types::ErrorObjectOwned::owned( -32000, format!("Database {} not found", db_id), @@ -183,13 +188,17 @@ impl RpcServerImpl { encryption_key: None, encrypt: false, backend: self.backend.clone(), + admin_secret: self.admin_secret.clone(), }; let mut server = Server::new(db_option).await; - // Set the selected database to the db_id for proper file naming + // Set the selected database to the db_id server.selected_db = db_id; + // Lazily open/create physical storage according to admin meta (per-db encryption) + let _ = server.current_storage(); + // Store the server let mut servers = self.servers.write().await; servers.insert(db_id, Arc::new(server.clone())); @@ -197,27 +206,10 @@ impl RpcServerImpl { Ok(Arc::new(server)) } - /// Discover existing database files in the base directory + /// Discover existing database IDs from admin DB 0 async fn discover_databases(&self) -> Vec { - let mut db_ids = Vec::new(); - - if let Ok(entries) = std::fs::read_dir(&self.base_dir) { - for entry in entries.flatten() { - if let Ok(file_name) = entry.file_name().into_string() { - // Check if it's a database file (ends with .db) - if file_name.ends_with(".db") { - // Extract database ID from filename (e.g., "11.db" -> 11) - if let Some(id_str) = file_name.strip_suffix(".db") { - if let Ok(db_id) = id_str.parse::() { - db_ids.push(db_id); - } - } - } - } - } - } - - db_ids + admin_meta::list_dbs(&self.base_dir, self.backend.clone(), &self.admin_secret) + .unwrap_or_default() } /// Get the next available database ID @@ -431,76 +423,52 @@ impl RpcServer for RpcServerImpl { async fn create_database( &self, backend: BackendType, - config: DatabaseConfig, + _config: DatabaseConfig, encryption_key: Option, ) -> RpcResult { - let db_id = self.get_next_db_id(encryption_key.is_some()).await; + // Allocate new ID via admin DB 0 + let db_id = admin_meta::allocate_next_id(&self.base_dir, self.backend.clone(), &self.admin_secret) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?; - // Handle both Redb and Sled backends - match backend { - BackendType::Redb | BackendType::Sled => { - // Create database directory - let db_dir = if let Some(path) = &config.storage_path { - std::path::PathBuf::from(path) - } else { - std::path::PathBuf::from(&self.base_dir).join(format!("rpc_db_{}", db_id)) - }; - - // Ensure directory exists - std::fs::create_dir_all(&db_dir) - .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned( - -32000, - format!("Failed to create directory: {}", e), - None::<()> - ))?; - - // Create DB options - let encrypt = encryption_key.is_some(); - let option = DBOption { - dir: db_dir.to_string_lossy().to_string(), - port: 0, // Not used for RPC-managed databases - debug: false, - encryption_key: encryption_key.clone(), - encrypt, - backend: match backend { - BackendType::Redb => crate::options::BackendType::Redb, - BackendType::Sled => crate::options::BackendType::Sled, - }, - }; - - // Create server instance - let mut server = Server::new(option).await; - - // Set the selected database to the db_id for proper file naming - server.selected_db = db_id; - - // Initialize the storage to create the database file - let _ = server.current_storage(); - - // Store the encryption key - { - let mut keys = self.encryption_keys.write().await; - keys.insert(db_id, encryption_key.clone()); - } - - // Initialize meta file - let meta = DatabaseMeta { - public: true, - keys: HashMap::new(), - }; - self.save_meta(db_id, &meta).await?; - - // Store the server - let mut servers = self.servers.write().await; - servers.insert(db_id, Arc::new(server)); - - Ok(db_id) - } + // Persist per-db encryption key in admin DB 0 if provided + if let Some(ref key) = encryption_key { + admin_meta::set_enc_key(&self.base_dir, self.backend.clone(), &self.admin_secret, db_id, key) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?; } + + // Ensure base dir exists + if let Err(e) = std::fs::create_dir_all(&self.base_dir) { + return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, format!("Failed to ensure base dir: {}", e), None::<()>)); + } + + // Create server instance using base_dir and admin secret + let option = DBOption { + dir: self.base_dir.clone(), + port: 0, // Not used for RPC-managed databases + debug: false, + encryption_key: None, // per-db key is stored in admin DB 0 + encrypt: false, // encryption decided per-db at open time + backend: match backend { + BackendType::Redb => crate::options::BackendType::Redb, + BackendType::Sled => crate::options::BackendType::Sled, + }, + admin_secret: self.admin_secret.clone(), + }; + + let mut server = Server::new(option).await; + server.selected_db = db_id; + + // Initialize storage to create physical .db with proper encryption from admin meta + let _ = server.current_storage(); + + // Store the server in cache + let mut servers = self.servers.write().await; + servers.insert(db_id, Arc::new(server)); + + Ok(db_id) } async fn set_encryption(&self, db_id: u64, _encryption_key: String) -> RpcResult { - // Note: In a real implementation, we'd need to modify the existing database // For now, return false as encryption can only be set during creation let _servers = self.servers.read().await; // TODO: Implement encryption setting for existing databases @@ -564,8 +532,6 @@ impl RpcServer for RpcServerImpl { } async fn add_access_key(&self, db_id: u64, key: String, permissions: String) -> RpcResult { - let mut meta = self.load_meta(db_id).await?; - let perms = match permissions.to_lowercase().as_str() { "read" => Permissions::Read, "readwrite" => Permissions::ReadWrite, @@ -576,52 +542,31 @@ impl RpcServer for RpcServerImpl { )), }; - let hash = hash_key(&key); - let access_key = AccessKey { - hash: hash.clone(), - permissions: perms, - created_at: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(), - }; - - meta.keys.insert(hash, access_key); - self.save_meta(db_id, &meta).await?; + admin_meta::add_access_key(&self.base_dir, self.backend.clone(), &self.admin_secret, db_id, &key, perms) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?; Ok(true) } async fn delete_access_key(&self, db_id: u64, key_hash: String) -> RpcResult { - let mut meta = self.load_meta(db_id).await?; - - if meta.keys.remove(&key_hash).is_some() { - // If no keys left, make database public - if meta.keys.is_empty() { - meta.public = true; - } - self.save_meta(db_id, &meta).await?; - Ok(true) - } else { - Ok(false) - } + let ok = admin_meta::delete_access_key(&self.base_dir, self.backend.clone(), &self.admin_secret, db_id, &key_hash) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?; + Ok(ok) } async fn list_access_keys(&self, db_id: u64) -> RpcResult> { - let meta = self.load_meta(db_id).await?; - let keys: Vec = meta.keys.values() - .map(|k| AccessKeyInfo { - hash: k.hash.clone(), - permissions: k.permissions.clone(), - created_at: k.created_at, - }) - .collect(); + let pairs = admin_meta::list_access_keys(&self.base_dir, self.backend.clone(), &self.admin_secret, db_id) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?; + let keys: Vec = pairs.into_iter().map(|(hash, perm, ts)| AccessKeyInfo { + hash, + permissions: perm, + created_at: ts, + }).collect(); Ok(keys) } async fn set_database_public(&self, db_id: u64, public: bool) -> RpcResult { - let mut meta = self.load_meta(db_id).await?; - meta.public = public; - self.save_meta(db_id, &meta).await?; + admin_meta::set_database_public(&self.base_dir, self.backend.clone(), &self.admin_secret, db_id, public) + .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?; Ok(true) } } \ No newline at end of file diff --git a/src/rpc_server.rs b/src/rpc_server.rs index 88ab432..eaabc5b 100644 --- a/src/rpc_server.rs +++ b/src/rpc_server.rs @@ -5,9 +5,9 @@ use jsonrpsee::RpcModule; use crate::rpc::{RpcServer, RpcServerImpl}; /// Start the RPC server on the specified address -pub async fn start_rpc_server(addr: SocketAddr, base_dir: String, backend: crate::options::BackendType) -> Result> { +pub async fn start_rpc_server(addr: SocketAddr, base_dir: String, backend: crate::options::BackendType, admin_secret: String) -> Result> { // Create the RPC server implementation - let rpc_impl = RpcServerImpl::new(base_dir, backend); + let rpc_impl = RpcServerImpl::new(base_dir, backend, admin_secret); // Create the RPC module let mut module = RpcModule::new(()); @@ -37,7 +37,7 @@ mod tests { let base_dir = "/tmp/test_rpc".to_string(); let backend = crate::options::BackendType::Redb; // Default for test - let handle = start_rpc_server(addr, base_dir, backend).await.unwrap(); + let handle = start_rpc_server(addr, base_dir, backend, "test-admin".to_string()).await.unwrap(); // Give the server a moment to start tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/src/server.rs b/src/server.rs index 63864c6..f02f065 100644 --- a/src/server.rs +++ b/src/server.rs @@ -11,9 +11,8 @@ use crate::cmd::Cmd; use crate::error::DBError; use crate::options; use crate::protocol::Protocol; -use crate::storage::Storage; -use crate::storage_sled::SledStorage; use crate::storage_trait::StorageBackend; +use crate::admin_meta; #[derive(Clone)] pub struct Server { @@ -58,50 +57,33 @@ impl Server { pub fn current_storage(&self) -> Result, DBError> { let mut cache = self.db_cache.write().unwrap(); - + if let Some(storage) = cache.get(&self.selected_db) { return Ok(storage.clone()); } - - - // Create new database file - let db_file_path = std::path::PathBuf::from(self.option.dir.clone()) - .join(format!("{}.db", self.selected_db)); - - // Ensure the directory exists before creating the database file - if let Some(parent_dir) = db_file_path.parent() { - std::fs::create_dir_all(parent_dir).map_err(|e| { - DBError(format!("Failed to create directory {}: {}", parent_dir.display(), e)) - })?; - } - - println!("Creating new db file: {}", db_file_path.display()); - - let storage: Arc = match self.option.backend { - options::BackendType::Redb => { - Arc::new(Storage::new( - db_file_path, - self.should_encrypt_db(self.selected_db), - self.option.encryption_key.as_deref() - )?) - } - options::BackendType::Sled => { - Arc::new(SledStorage::new( - db_file_path, - self.should_encrypt_db(self.selected_db), - self.option.encryption_key.as_deref() - )?) - } + + // Use process-wide shared handles to avoid sled/reDB double-open lock contention. + let storage = if self.selected_db == 0 { + // Admin DB 0: always via singleton + admin_meta::open_admin_storage( + &self.option.dir, + self.option.backend.clone(), + &self.option.admin_secret, + )? + } else { + // Data DBs: via global registry keyed by id + admin_meta::open_data_storage( + &self.option.dir, + self.option.backend.clone(), + &self.option.admin_secret, + self.selected_db, + )? }; - + cache.insert(self.selected_db, storage.clone()); Ok(storage) } - fn should_encrypt_db(&self, db_index: u64) -> bool { - // DB 0-9 are non-encrypted, DB 10+ are encrypted - self.option.encrypt && db_index >= 10 - } /// Check if current permissions allow read operations pub fn has_read_permission(&self) -> bool { diff --git a/tests/debug_hset.rs b/tests/debug_hset.rs index 7930be8..44cd39b 100644 --- a/tests/debug_hset.rs +++ b/tests/debug_hset.rs @@ -28,6 +28,7 @@ async fn debug_hset_simple() { encrypt: false, encryption_key: None, backend: herodb::options::BackendType::Redb, + admin_secret: "test-admin".to_string(), }; let mut server = Server::new(option).await; @@ -48,6 +49,12 @@ async fn debug_hset_simple() { sleep(Duration::from_millis(200)).await; let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)).await.unwrap(); + // Acquire ReadWrite permissions on this connection + let resp = send_command( + &mut stream, + "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n", + ).await; + assert!(resp.contains("OK"), "Failed SELECT handshake: {}", resp); // Test simple HSET println!("Testing HSET..."); diff --git a/tests/debug_hset_simple.rs b/tests/debug_hset_simple.rs index 356e704..fe99f0f 100644 --- a/tests/debug_hset_simple.rs +++ b/tests/debug_hset_simple.rs @@ -19,6 +19,7 @@ async fn debug_hset_return_value() { encrypt: false, encryption_key: None, backend: herodb::options::BackendType::Redb, + admin_secret: "test-admin".to_string(), }; let mut server = Server::new(option).await; @@ -40,12 +41,19 @@ async fn debug_hset_return_value() { // Connect and test HSET let mut stream = TcpStream::connect("127.0.0.1:16390").await.unwrap(); + + // Acquire ReadWrite permissions for this new connection + let handshake = "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n"; + stream.write_all(handshake.as_bytes()).await.unwrap(); + let mut buffer = [0; 1024]; + let n = stream.read(&mut buffer).await.unwrap(); + let resp = String::from_utf8_lossy(&buffer[..n]); + assert!(resp.contains("OK"), "Failed SELECT handshake: {}", resp); // Send HSET command let cmd = "*4\r\n$4\r\nHSET\r\n$4\r\nhash\r\n$6\r\nfield1\r\n$6\r\nvalue1\r\n"; stream.write_all(cmd.as_bytes()).await.unwrap(); - let mut buffer = [0; 1024]; let n = stream.read(&mut buffer).await.unwrap(); let response = String::from_utf8_lossy(&buffer[..n]); diff --git a/tests/redis_integration_tests.rs b/tests/redis_integration_tests.rs index 47033e1..a017780 100644 --- a/tests/redis_integration_tests.rs +++ b/tests/redis_integration_tests.rs @@ -12,7 +12,15 @@ fn get_redis_connection(port: u16) -> Connection { match client.get_connection() { Ok(mut conn) => { if redis::cmd("PING").query::(&mut conn).is_ok() { - return conn; + // Acquire ReadWrite permissions on this connection + let sel: RedisResult = redis::cmd("SELECT") + .arg(0) + .arg("KEY") + .arg("test-admin") + .query(&mut conn); + if sel.is_ok() { + return conn; + } } } Err(e) => { @@ -78,6 +86,8 @@ fn setup_server() -> (ServerProcessGuard, u16) { "--port", &port.to_string(), "--debug", + "--admin-secret", + "test-admin", ]) .spawn() .expect("Failed to start server process"); diff --git a/tests/redis_tests.rs b/tests/redis_tests.rs index f6e8a13..724704c 100644 --- a/tests/redis_tests.rs +++ b/tests/redis_tests.rs @@ -23,18 +23,29 @@ async fn start_test_server(test_name: &str) -> (Server, u16) { encrypt: false, encryption_key: None, backend: herodb::options::BackendType::Redb, + admin_secret: "test-admin".to_string(), }; let server = Server::new(option).await; (server, port) } -// Helper function to connect to the test server + // Helper function to connect to the test server async fn connect_to_server(port: u16) -> TcpStream { let mut attempts = 0; loop { match TcpStream::connect(format!("127.0.0.1:{}", port)).await { - Ok(stream) => return stream, + Ok(mut stream) => { + // Obtain ReadWrite permissions for this connection by selecting DB 0 with admin key + let resp = send_command( + &mut stream, + "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n", + ).await; + if !resp.contains("OK") { + panic!("Failed to acquire write permissions via SELECT 0 KEY test-admin: {}", resp); + } + return stream; + } Err(_) if attempts < 10 => { attempts += 1; sleep(Duration::from_millis(100)).await; diff --git a/tests/simple_integration_test.rs b/tests/simple_integration_test.rs index 42269df..706c9cb 100644 --- a/tests/simple_integration_test.rs +++ b/tests/simple_integration_test.rs @@ -25,6 +25,7 @@ async fn start_test_server(test_name: &str) -> (Server, u16) { encrypt: false, encryption_key: None, backend: herodb::options::BackendType::Redb, + admin_secret: "test-admin".to_string(), }; let server = Server::new(option).await; @@ -34,9 +35,16 @@ async fn start_test_server(test_name: &str) -> (Server, u16) { // Helper function to send Redis command and get response async fn send_redis_command(port: u16, command: &str) -> String { let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)).await.unwrap(); + + // Acquire ReadWrite permissions on this new connection + let handshake = "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n"; + stream.write_all(handshake.as_bytes()).await.unwrap(); + let mut buffer = [0; 1024]; + let _ = stream.read(&mut buffer).await.unwrap(); // Read and ignore the OK for handshake + + // Now send the intended command stream.write_all(command.as_bytes()).await.unwrap(); - let mut buffer = [0; 1024]; let n = stream.read(&mut buffer).await.unwrap(); String::from_utf8_lossy(&buffer[..n]).to_string() } @@ -184,12 +192,19 @@ async fn test_transaction_operations() { sleep(Duration::from_millis(100)).await; - // Use a single connection for the transaction + // Use a single connection for the transaction let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)).await.unwrap(); + // Acquire write permissions for this connection + let handshake = "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n"; + stream.write_all(handshake.as_bytes()).await.unwrap(); + let mut buffer = [0; 1024]; + let n = stream.read(&mut buffer).await.unwrap(); + let resp = String::from_utf8_lossy(&buffer[..n]); + assert!(resp.contains("OK")); + // Test MULTI stream.write_all("*1\r\n$5\r\nMULTI\r\n".as_bytes()).await.unwrap(); - let mut buffer = [0; 1024]; let n = stream.read(&mut buffer).await.unwrap(); let response = String::from_utf8_lossy(&buffer[..n]); assert!(response.contains("OK")); diff --git a/tests/simple_redis_test.rs b/tests/simple_redis_test.rs index 8afb304..cd8c0a7 100644 --- a/tests/simple_redis_test.rs +++ b/tests/simple_redis_test.rs @@ -23,6 +23,7 @@ async fn start_test_server(test_name: &str) -> (Server, u16) { encrypt: false, encryption_key: None, backend: herodb::options::BackendType::Redb, + admin_secret: "test-admin".to_string(), }; let server = Server::new(option).await; @@ -38,12 +39,22 @@ async fn send_command(stream: &mut TcpStream, command: &str) -> String { String::from_utf8_lossy(&buffer[..n]).to_string() } -// Helper function to connect to the test server + // Helper function to connect to the test server async fn connect_to_server(port: u16) -> TcpStream { let mut attempts = 0; loop { match TcpStream::connect(format!("127.0.0.1:{}", port)).await { - Ok(stream) => return stream, + Ok(mut stream) => { + // Acquire ReadWrite permissions for this connection + let resp = send_command( + &mut stream, + "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n", + ).await; + if !resp.contains("OK") { + panic!("Failed to acquire write permissions via SELECT 0 KEY test-admin: {}", resp); + } + return stream; + } Err(_) if attempts < 10 => { attempts += 1; sleep(Duration::from_millis(100)).await; @@ -97,14 +108,21 @@ async fn test_hset_clean_db() { sleep(Duration::from_millis(200)).await; let mut stream = connect_to_server(port).await; - - // Test HSET - should return 1 for new field - let response = send_command(&mut stream, "*4\r\n$4\r\nHSET\r\n$4\r\nhash\r\n$6\r\nfield1\r\n$6\r\nvalue1\r\n").await; + + // Ensure clean DB state (admin DB 0 may be shared due to global singleton) + let flush = send_command(&mut stream, "*1\r\n$7\r\nFLUSHDB\r\n").await; + assert!(flush.contains("OK"), "Failed to FLUSHDB: {}", flush); + + // Test HSET - should return 1 for new field (use a unique key name to avoid collisions) + let key = "hash_clean"; + let hset_cmd = format!("*4\r\n$4\r\nHSET\r\n${}\r\n{}\r\n$6\r\nfield1\r\n$6\r\nvalue1\r\n", key.len(), key); + let response = send_command(&mut stream, &hset_cmd).await; println!("HSET response: {}", response); assert!(response.contains("1"), "Expected HSET to return 1, got: {}", response); // Test HGET - let response = send_command(&mut stream, "*3\r\n$4\r\nHGET\r\n$4\r\nhash\r\n$6\r\nfield1\r\n").await; + let hget_cmd = format!("*3\r\n$4\r\nHGET\r\n${}\r\n{}\r\n$6\r\nfield1\r\n", key.len(), key); + let response = send_command(&mut stream, &hget_cmd).await; println!("HGET response: {}", response); assert!(response.contains("value1")); } diff --git a/tests/usage_suite.rs b/tests/usage_suite.rs index 9a1af17..0203c28 100644 --- a/tests/usage_suite.rs +++ b/tests/usage_suite.rs @@ -23,6 +23,7 @@ async fn start_test_server(test_name: &str) -> (Server, u16) { encrypt: false, encryption_key: None, backend: herodb::options::BackendType::Redb, + admin_secret: "test-admin".to_string(), }; let server = Server::new(option).await; @@ -61,7 +62,17 @@ async fn connect(port: u16) -> TcpStream { let mut attempts = 0; loop { match TcpStream::connect(format!("127.0.0.1:{}", port)).await { - Ok(s) => return s, + Ok(mut s) => { + // Acquire ReadWrite permissions for this connection using admin DB 0 + let resp = send_cmd(&mut s, &["SELECT", "0", "KEY", "test-admin"]).await; + assert_contains(&resp, "OK", "SELECT 0 KEY test-admin handshake"); + + // Ensure clean slate per test on DB 0 + let fl = send_cmd(&mut s, &["FLUSHDB"]).await; + assert_contains(&fl, "OK", "FLUSHDB after handshake"); + + return s; + } Err(_) if attempts < 30 => { attempts += 1; sleep(Duration::from_millis(100)).await; @@ -246,9 +257,9 @@ async fn test_01_connection_and_info() { let getname = send_cmd(&mut s, &["CLIENT", "GETNAME"]).await; assert_contains(&getname, "myapp", "CLIENT GETNAME"); - // SELECT db - let sel = send_cmd(&mut s, &["SELECT", "0"]).await; - assert_contains(&sel, "OK", "SELECT 0"); + // SELECT db (requires key on DB 0) + let sel = send_cmd(&mut s, &["SELECT", "0", "KEY", "test-admin"]).await; + assert_contains(&sel, "OK", "SELECT 0 with key"); // QUIT should close connection after sending OK let quit = send_cmd(&mut s, &["QUIT"]).await; -- 2.43.0