diff --git a/Cargo.lock b/Cargo.lock index 86db035..80b50ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,15 +197,6 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "2.9.2" @@ -221,12 +212,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.10.1" @@ -661,28 +646,22 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "herocrypto" +version = "0.1.0" + [[package]] name = "herodb" -version = "0.0.1" +version = "0.1.0" dependencies = [ - "age", "anyhow", - "base64 0.22.1", - "bincode", - "byteorder", "bytes", - "chacha20poly1305", "clap", - "ed25519-dalek", - "futures", - "rand", - "redb", + "libcryptoa", + "libdbstorage", + "log", "redis", - "secrecy", "serde", - "serde_json", - "sha2", - "thiserror", "tokio", ] @@ -949,6 +928,39 @@ version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +[[package]] +name = "libcrypto" +version = "0.1.0" +dependencies = [ + "chacha20poly1305", + "libdbstorage", + "rand", + "sha2", + "thiserror", +] + +[[package]] +name = "libcryptoa" +version = "0.1.0" +dependencies = [ + "age", + "base64 0.22.1", + "ed25519-dalek", + "rand", + "secrecy", + "thiserror", +] + +[[package]] +name = "libdbstorage" +version = "0.1.0" +dependencies = [ + "redb", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "litemap" version = "0.8.0" @@ -1530,6 +1542,10 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" name = "supervisor" version = "0.1.0" +[[package]] +name = "supervisorrpc" +version = "0.1.0" + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index 9ca5311..4484630 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,37 @@ [workspace] -members = [ - "herodb", - "supervisor", -] resolver = "2" +members = [ + "crates/herodb", + "crates/libdbstorage", + "crates/libcrypto", + "crates/libcryptoa", + "crates/herocrypto", + "crates/supervisorrpc", + "crates/supervisor", +] + +[workspace.dependencies] +# Central place for dependencies shared across the workspace +anyhow = "1.0" +tokio = { version = "1", features = ["full"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "1.0" +log = "0.4" + +# Crypto deps +chacha20poly1305 = "0.10" +rand = "0.8" +sha2 = "0.10" +age = "0.10" +secrecy = "0.8" +ed25519-dalek = "2" +base64 = "0.22" + +# DB +redb = "2.1" -# You can define shared profiles for all workspace members here [profile.release] lto = true -codegen-units = 1 +codegen-units =1 strip = true \ No newline at end of file diff --git a/crates/herocrypto/Cargo.toml b/crates/herocrypto/Cargo.toml new file mode 100644 index 0000000..d34c159 --- /dev/null +++ b/crates/herocrypto/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "herocrypto" +version = "0.1.0" +edition = "2024" + +[dependencies] diff --git a/crates/herocrypto/src/lib.rs b/crates/herocrypto/src/lib.rs new file mode 100644 index 0000000..b93cf3f --- /dev/null +++ b/crates/herocrypto/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/crates/herodb/Cargo.toml b/crates/herodb/Cargo.toml new file mode 100644 index 0000000..be59310 --- /dev/null +++ b/crates/herodb/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "herodb" +version = "0.1.0" +authors = ["Pin Fang "] +edition = "2021" + +# THIS IS A BINARY, NOT A LIBRARY +[[bin]] +name = "herodb" +path = "src/main.rs" + +[dependencies] +# Workspace dependencies +anyhow = { workspace = true } +tokio = { workspace = true } +serde = { workspace = true } +log = { workspace = true } + +# Local Crate Dependencies +libdbstorage = { path = "../libdbstorage" } +libcryptoa = { path = "../libcryptoa" } + +# Other dependencies +clap = { version = "4.5", features = ["derive"] } +bytes = "1.3.0" # Example, keep specific versions if needed + +[dev-dependencies] +redis = { version = "0.24", features = ["aio", "tokio-comp"] } \ No newline at end of file diff --git a/herodb/README.md b/crates/herodb/README.md similarity index 100% rename from herodb/README.md rename to crates/herodb/README.md diff --git a/herodb/src/age.rs b/crates/herodb/age.rs similarity index 100% rename from herodb/src/age.rs rename to crates/herodb/age.rs diff --git a/herodb/src/cmd.rs b/crates/herodb/cmd.rs similarity index 100% rename from herodb/src/cmd.rs rename to crates/herodb/cmd.rs diff --git a/herodb/src/crypto.rs b/crates/herodb/crypto.rs similarity index 100% rename from herodb/src/crypto.rs rename to crates/herodb/crypto.rs diff --git a/herodb/src/error.rs b/crates/herodb/error.rs similarity index 100% rename from herodb/src/error.rs rename to crates/herodb/error.rs diff --git a/herodb/examples/age_bash_demo.sh b/crates/herodb/examples/age_bash_demo.sh similarity index 100% rename from herodb/examples/age_bash_demo.sh rename to crates/herodb/examples/age_bash_demo.sh diff --git a/herodb/examples/age_persist_demo.rs b/crates/herodb/examples/age_persist_demo.rs similarity index 100% rename from herodb/examples/age_persist_demo.rs rename to crates/herodb/examples/age_persist_demo.rs diff --git a/herodb/src/lib.rs b/crates/herodb/lib.rs similarity index 100% rename from herodb/src/lib.rs rename to crates/herodb/lib.rs diff --git a/herodb/src/main.rs b/crates/herodb/main.rs similarity index 100% rename from herodb/src/main.rs rename to crates/herodb/main.rs diff --git a/herodb/src/options.rs b/crates/herodb/options.rs similarity index 100% rename from herodb/src/options.rs rename to crates/herodb/options.rs diff --git a/herodb/src/protocol.rs b/crates/herodb/protocol.rs similarity index 100% rename from herodb/src/protocol.rs rename to crates/herodb/protocol.rs diff --git a/herodb/src/server.rs b/crates/herodb/server.rs similarity index 100% rename from herodb/src/server.rs rename to crates/herodb/server.rs diff --git a/crates/herodb/src/main.rs b/crates/herodb/src/main.rs new file mode 100644 index 0000000..e71fdf5 --- /dev/null +++ b/crates/herodb/src/main.rs @@ -0,0 +1 @@ +fn main() {} \ No newline at end of file diff --git a/herodb/src/storage/mod.rs b/crates/herodb/storage/mod.rs similarity index 100% rename from herodb/src/storage/mod.rs rename to crates/herodb/storage/mod.rs diff --git a/herodb/src/storage/storage_basic.rs b/crates/herodb/storage/storage_basic.rs similarity index 100% rename from herodb/src/storage/storage_basic.rs rename to crates/herodb/storage/storage_basic.rs diff --git a/herodb/src/storage/storage_extra.rs b/crates/herodb/storage/storage_extra.rs similarity index 100% rename from herodb/src/storage/storage_extra.rs rename to crates/herodb/storage/storage_extra.rs diff --git a/herodb/src/storage/storage_hset.rs b/crates/herodb/storage/storage_hset.rs similarity index 100% rename from herodb/src/storage/storage_hset.rs rename to crates/herodb/storage/storage_hset.rs diff --git a/herodb/src/storage/storage_lists.rs b/crates/herodb/storage/storage_lists.rs similarity index 100% rename from herodb/src/storage/storage_lists.rs rename to crates/herodb/storage/storage_lists.rs diff --git a/herodb/tests/debug_hset.rs b/crates/herodb/tests/debug_hset.rs similarity index 100% rename from herodb/tests/debug_hset.rs rename to crates/herodb/tests/debug_hset.rs diff --git a/herodb/tests/debug_hset_simple.rs b/crates/herodb/tests/debug_hset_simple.rs similarity index 100% rename from herodb/tests/debug_hset_simple.rs rename to crates/herodb/tests/debug_hset_simple.rs diff --git a/herodb/tests/debug_protocol.rs b/crates/herodb/tests/debug_protocol.rs similarity index 100% rename from herodb/tests/debug_protocol.rs rename to crates/herodb/tests/debug_protocol.rs diff --git a/herodb/tests/redis_integration_tests.rs b/crates/herodb/tests/redis_integration_tests.rs similarity index 100% rename from herodb/tests/redis_integration_tests.rs rename to crates/herodb/tests/redis_integration_tests.rs diff --git a/herodb/tests/redis_tests.rs b/crates/herodb/tests/redis_tests.rs similarity index 100% rename from herodb/tests/redis_tests.rs rename to crates/herodb/tests/redis_tests.rs diff --git a/herodb/tests/simple_integration_test.rs b/crates/herodb/tests/simple_integration_test.rs similarity index 100% rename from herodb/tests/simple_integration_test.rs rename to crates/herodb/tests/simple_integration_test.rs diff --git a/herodb/tests/simple_redis_test.rs b/crates/herodb/tests/simple_redis_test.rs similarity index 100% rename from herodb/tests/simple_redis_test.rs rename to crates/herodb/tests/simple_redis_test.rs diff --git a/crates/libcrypto/Cargo.toml b/crates/libcrypto/Cargo.toml new file mode 100644 index 0000000..dac73b4 --- /dev/null +++ b/crates/libcrypto/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "libcrypto" +version = "0.1.0" +edition = "2021" + +[dependencies] +chacha20poly1305 = "0.10" +rand = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } + +# Add this dependency for the From for DBError +libdbstorage = { path = "../libdbstorage", optional = true } + +[features] +storage_compat = ["dep:libdbstorage"] \ No newline at end of file diff --git a/crates/libcrypto/src/lib.rs b/crates/libcrypto/src/lib.rs new file mode 100644 index 0000000..b93cf3f --- /dev/null +++ b/crates/libcrypto/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/crates/libcryptoa/Cargo.toml b/crates/libcryptoa/Cargo.toml new file mode 100644 index 0000000..ffe544b --- /dev/null +++ b/crates/libcryptoa/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "libcryptoa" +version = "0.1.0" +edition = "2021" + +[dependencies] +age = { workspace = true } +secrecy = { workspace = true } +ed25519-dalek = { workspace = true } +base64 = { workspace = true } +rand = { workspace = true } +thiserror = { workspace = true } \ No newline at end of file diff --git a/crates/libcryptoa/src/lib.rs b/crates/libcryptoa/src/lib.rs new file mode 100644 index 0000000..b93cf3f --- /dev/null +++ b/crates/libcryptoa/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/crates/libdbstorage/Cargo.toml b/crates/libdbstorage/Cargo.toml new file mode 100644 index 0000000..cfec9a4 --- /dev/null +++ b/crates/libdbstorage/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "libdbstorage" +version = "0.1.0" +edition = "2021" + +[dependencies] +redb = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } + +[dev-dependencies] +thiserror = { workspace = true } \ No newline at end of file diff --git a/crates/libdbstorage/src/lib.rs b/crates/libdbstorage/src/lib.rs new file mode 100644 index 0000000..b93cf3f --- /dev/null +++ b/crates/libdbstorage/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/supervisor/Cargo.toml b/crates/supervisor/Cargo.toml similarity index 100% rename from supervisor/Cargo.toml rename to crates/supervisor/Cargo.toml diff --git a/supervisor/src/main.rs b/crates/supervisor/src/main.rs similarity index 100% rename from supervisor/src/main.rs rename to crates/supervisor/src/main.rs diff --git a/crates/supervisorrpc/Cargo.toml b/crates/supervisorrpc/Cargo.toml new file mode 100644 index 0000000..4413902 --- /dev/null +++ b/crates/supervisorrpc/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "supervisorrpc" +version = "0.1.0" +edition = "2024" + +[dependencies] diff --git a/crates/supervisorrpc/src/main.rs b/crates/supervisorrpc/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/crates/supervisorrpc/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/herodb/Cargo.toml b/herodb/Cargo.toml deleted file mode 100644 index b71eb22..0000000 --- a/herodb/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "herodb" -version = "0.0.1" -authors = ["Pin Fang "] -edition = "2021" - -[dependencies] -anyhow = "1.0.59" -bytes = "1.3.0" -thiserror = "1.0.32" -tokio = { version = "1.23.0", features = ["full"] } -clap = { version = "4.5.20", features = ["derive"] } -byteorder = "1.4.3" -futures = "0.3" -redb = "2.1.3" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -bincode = "1.3.3" -chacha20poly1305 = "0.10.1" -rand = "0.8" -sha2 = "0.10" -age = "0.10" -secrecy = "0.8" -ed25519-dalek = "2" -base64 = "0.22" - -[dev-dependencies] -redis = { version = "0.24", features = ["aio", "tokio-comp"] } diff --git a/herodb/instructions/encrypt.md b/herodb/instructions/encrypt.md deleted file mode 100644 index f68424a..0000000 --- a/herodb/instructions/encrypt.md +++ /dev/null @@ -1,99 +0,0 @@ - -### Cargo.toml - -```toml -[dependencies] -chacha20poly1305 = { version = "0.10", features = ["xchacha20"] } -rand = "0.8" -sha2 = "0.10" -``` - -### `crypto_factory.rs` - -```rust -use chacha20poly1305::{ - aead::{Aead, KeyInit, OsRng}, - XChaCha20Poly1305, Key, XNonce, -}; -use rand::RngCore; -use sha2::{Digest, Sha256}; - -const VERSION: u8 = 1; -const NONCE_LEN: usize = 24; -const TAG_LEN: usize = 16; - -#[derive(Debug)] -pub enum CryptoError { - Format, // wrong length / header - Version(u8), // unknown version - Decrypt, // wrong key or corrupted data -} - -/// Super-simple factory: new(secret) + encrypt(bytes) + decrypt(bytes) -pub struct CryptoFactory { - key: Key, -} - -impl CryptoFactory { - /// Accepts any secret bytes; turns them into a 32-byte key (SHA-256). - /// (If your secret is already 32 bytes, this is still fine.) - pub fn new>(secret: S) -> Self { - let mut h = Sha256::new(); - h.update(b"xchacha20poly1305-factory:v1"); // domain separation - h.update(secret.as_ref()); - let digest = h.finalize(); // 32 bytes - let key = Key::::from_slice(&digest).to_owned(); - Self { key } - } - - /// Output layout: [version:1][nonce:24][ciphertext||tag] - pub fn encrypt(&self, plaintext: &[u8]) -> Vec { - let cipher = XChaCha20Poly1305::new(&self.key); - - let mut nonce_bytes = [0u8; NONCE_LEN]; - OsRng.fill_bytes(&mut nonce_bytes); - let nonce = XNonce::from_slice(&nonce_bytes); - - let mut out = Vec::with_capacity(1 + NONCE_LEN + plaintext.len() + TAG_LEN); - out.push(VERSION); - out.extend_from_slice(&nonce_bytes); - - let ct = cipher.encrypt(nonce, plaintext).expect("encrypt"); - out.extend_from_slice(&ct); - out - } - - pub fn decrypt(&self, blob: &[u8]) -> Result, CryptoError> { - if blob.len() < 1 + NONCE_LEN + TAG_LEN { - return Err(CryptoError::Format); - } - let ver = blob[0]; - if ver != VERSION { - return Err(CryptoError::Version(ver)); - } - - let nonce = XNonce::from_slice(&blob[1..1 + NONCE_LEN]); - let ct = &blob[1 + NONCE_LEN..]; - - let cipher = XChaCha20Poly1305::new(&self.key); - cipher.decrypt(nonce, ct).map_err(|_| CryptoError::Decrypt) - } -} -``` - -### Tiny usage example - -```rust -fn main() { - let f = CryptoFactory::new(b"super-secret-key-material"); - let val = b"\x00\xFFbinary\x01\x02\x03"; - - let blob = f.encrypt(val); - let roundtrip = f.decrypt(&blob).unwrap(); - - assert_eq!(roundtrip, val); -} -``` - -That’s it: `new(secret)`, `encrypt(bytes)`, `decrypt(bytes)`. -You can stash the returned `blob` directly in your storage layer behind Redis. diff --git a/herodb/instructions/redb.md b/herodb/instructions/redb.md deleted file mode 100644 index 27121c1..0000000 --- a/herodb/instructions/redb.md +++ /dev/null @@ -1,80 +0,0 @@ -======================== -CODE SNIPPETS -======================== -TITLE: 1PC+C Commit Strategy Vulnerability Example -DESCRIPTION: Illustrates a scenario where a partially committed transaction might appear complete due to the non-cryptographic checksum (XXH3) used in the 1PC+C commit strategy. This requires controlling page flush order, introducing a crash during fsync, and ensuring valid checksums for partially written data. - -SOURCE: https://github.com/cberner/redb/blob/master/docs/design.md#_snippet_9 - -LANGUAGE: rust -CODE: -``` -table.insert(malicious_key, malicious_value); -table.insert(good_key, good_value); -txn.commit(); -``` - -LANGUAGE: rust -CODE: -``` -table.insert(malicious_key, malicious_value); -txn.commit(); -``` - ----------------------------------------- - -TITLE: Basic Key-Value Operations in redb -DESCRIPTION: Demonstrates the fundamental usage of redb for creating a database, opening a table, inserting a key-value pair, and retrieving the value within separate read and write transactions. - -SOURCE: https://github.com/cberner/redb/blob/master/README.md#_snippet_0 - -LANGUAGE: rust -CODE: -``` -use redb::{Database, Error, ReadableTable, TableDefinition}; - -const TABLE: TableDefinition<&str, u64> = TableDefinition::new("my_data"); - -fn main() -> Result<(), Error> { - let db = Database::create("my_db.redb")?; - let write_txn = db.begin_write()?; - { - let mut table = write_txn.open_table(TABLE)?; - table.insert("my_key", &123)?; - } - write_txn.commit()?; - - let read_txn = db.begin_read()?; - let table = read_txn.open_table(TABLE)?; - assert_eq!(table.get("my_key")?.unwrap().value(), 123); - - Ok(()) -} -``` - - - -## What *redb* currently supports: - -* Simple operations like creating databases, inserting key-value pairs, opening and reading tables ([GitHub][1]). -* No mention of operations such as: - - * Iterating over keys with a given prefix. - * Range queries based on string prefixes. - * Specialized prefix‑filtered lookups. - - -## implement range scans as follows - -You can implement prefix-like functionality using **range scans** combined with manual checks, similar to using a `BTreeSet` in Rust: - -```rust -for key in table.range(prefix..).keys() { - if !key.starts_with(prefix) { - break; - } - // process key -} -``` - -This pattern iterates keys starting at the prefix, and stops once a key no longer matches the prefix—this works because the keys are sorted ([GitHub][1]). diff --git a/herodb/instructions/redis_basic_client.md b/herodb/instructions/redis_basic_client.md deleted file mode 100644 index f6a90e4..0000000 --- a/herodb/instructions/redis_basic_client.md +++ /dev/null @@ -1,150 +0,0 @@ -] -# INFO - -**What it does** -Returns server stats in a human-readable text block, optionally filtered by sections. Typical sections: `server`, `clients`, `memory`, `persistence`, `stats`, `replication`, `cpu`, `commandstats`, `latencystats`, `cluster`, `modules`, `keyspace`, `errorstats`. Special args: `all`, `default`, `everything`. The reply is a **Bulk String** with `#
` headers and `key:value` lines. ([Redis][1]) - -**Syntax** - -``` -INFO [section [section ...]] -``` - -**Return (RESP2/RESP3)**: Bulk String. ([Redis][1]) - -**RESP request/response** - -``` -# Request: whole default set -*1\r\n$4\r\nINFO\r\n - -# Request: a specific section, e.g., clients -*2\r\n$4\r\nINFO\r\n$7\r\nclients\r\n - -# Response (prefix shown; body is long) -$1234\r\n# Server\r\nredis_version:7.4.0\r\n...\r\n# Clients\r\nconnected_clients:3\r\n...\r\n -``` - -(Reply type/format per RESP spec and the INFO page.) ([Redis][2]) - ---- - -# Connection “name” (there is **no** top-level `NAME` command) - -Redis doesn’t have a standalone `NAME` command. Connection names are handled via `CLIENT SETNAME` and retrieved via `CLIENT GETNAME`. ([Redis][3]) - -## CLIENT SETNAME - -Assigns a human label to the current connection (shown in `CLIENT LIST`, logs, etc.). No spaces allowed in the name; empty string clears it. Length is limited by Redis string limits (practically huge). **Reply**: Simple String `OK`. ([Redis][4]) - -**Syntax** - -``` -CLIENT SETNAME connection-name -``` - -**RESP** - -``` -# Set the name "myapp" -*3\r\n$6\r\nCLIENT\r\n$7\r\nSETNAME\r\n$5\r\nmyapp\r\n - -# Reply -+OK\r\n -``` - -## CLIENT GETNAME - -Returns the current connection’s name or **Null Bulk String** if unset. ([Redis][5]) - -**Syntax** - -``` -CLIENT GETNAME -``` - -**RESP** - -``` -# Before SETNAME: -*2\r\n$6\r\nCLIENT\r\n$7\r\nGETNAME\r\n -$-1\r\n # nil (no name) - -# After SETNAME myapp: -*2\r\n$6\r\nCLIENT\r\n$7\r\nGETNAME\r\n -$5\r\nmyapp\r\n -``` - -(Null/Bulk String encoding per RESP spec.) ([Redis][2]) - ---- - -# CLIENT (container command + key subcommands) - -`CLIENT` is a **container**; use subcommands like `CLIENT LIST`, `CLIENT INFO`, `CLIENT ID`, `CLIENT KILL`, `CLIENT TRACKING`, etc. Call `CLIENT HELP` to enumerate them. ([Redis][3]) - -## CLIENT LIST - -Shows all connections as a single **Bulk String**: one line per client with `field=value` pairs (includes `id`, `addr`, `name`, `db`, `user`, `resp`, and more). Filters: `TYPE` and `ID`. **Return**: Bulk String (RESP2/RESP3). ([Redis][6]) - -**Syntax** - -``` -CLIENT LIST [TYPE ] [ID client-id ...] -``` - -**RESP** - -``` -*2\r\n$6\r\nCLIENT\r\n$4\r\nLIST\r\n - -# Reply (single Bulk String; example with one line shown) -$188\r\nid=7 addr=127.0.0.1:60840 laddr=127.0.0.1:6379 fd=8 name=myapp age=12 idle=3 flags=N db=0 ...\r\n -``` - -## CLIENT INFO - -Returns info for **this** connection only (same format/fields as a single line of `CLIENT LIST`). **Return**: Bulk String. Available since 6.2.0. ([Redis][7]) - -**Syntax** - -``` -CLIENT INFO -``` - -**RESP** - -``` -*2\r\n$6\r\nCLIENT\r\n$4\r\nINFO\r\n - -$160\r\nid=7 addr=127.0.0.1:60840 laddr=127.0.0.1:6379 fd=8 name=myapp db=0 user=default resp=2 ...\r\n -``` - ---- - -# RESP notes you’ll need for your parser - -* **Requests** are Arrays: `*N\r\n` followed by `N` Bulk Strings for verb/args. -* **Common replies here**: Simple String (`+OK\r\n`), Bulk String (`$\r\n...\r\n`), and **Null Bulk String** (`$-1\r\n`). (These cover `INFO`, `CLIENT LIST/INFO`, `CLIENT GETNAME`, `CLIENT SETNAME`.) ([Redis][2]) - ---- - -## Sources (checked) - -* INFO command (syntax, sections, behavior). ([Redis][1]) -* RESP spec (request/response framing, Bulk/Null Bulk Strings). ([Redis][2]) -* CLIENT container + subcommands index. ([Redis][3]) -* CLIENT LIST (fields, bulk-string return, filters). ([Redis][6]) -* CLIENT INFO (exists since 6.2, reply format). ([Redis][7]) -* CLIENT SETNAME (no spaces; clears with empty string; huge length OK). ([Redis][4]) -* CLIENT GETNAME (nil if unset). ([Redis][5]) - -If you want, I can fold this into a tiny Rust “command + RESP” test harness that exercises `INFO`, `CLIENT SETNAME/GETNAME`, `CLIENT LIST`, and `CLIENT INFO` against your in-mem RESP parser. - -[1]: https://redis.io/docs/latest/commands/info/ "INFO | Docs" -[2]: https://redis.io/docs/latest/develop/reference/protocol-spec/?utm_source=chatgpt.com "Redis serialization protocol specification | Docs" -[3]: https://redis.io/docs/latest/commands/client/ "CLIENT | Docs" -[4]: https://redis.io/docs/latest/commands/client-setname/?utm_source=chatgpt.com "CLIENT SETNAME | Docs" -[5]: https://redis.io/docs/latest/commands/client-getname/?utm_source=chatgpt.com "CLIENT GETNAME | Docs" -[6]: https://redis.io/docs/latest/commands/client-list/ "CLIENT LIST | Docs" -[7]: https://redis.io/docs/latest/commands/client-info/?utm_source=chatgpt.com "CLIENT INFO | Docs" diff --git a/herodb/instructions/redis_basics.md b/herodb/instructions/redis_basics.md deleted file mode 100644 index 302dd1e..0000000 --- a/herodb/instructions/redis_basics.md +++ /dev/null @@ -1,251 +0,0 @@ -Got it 👍 — let’s break this down properly. - -Redis has two broad classes you’re asking about: - -1. **Basic key-space functions** (SET, GET, DEL, EXISTS, etc.) -2. **Iteration commands** (`SCAN`, `SSCAN`, `HSCAN`, `ZSCAN`) - -And for each I’ll show: - -* What it does -* How it works at a high level -* Its **RESP protocol implementation** (the actual wire format). - ---- - -# 1. Basic Key-Space Commands - -### `SET key value` - -* Stores a string value at a key. -* Overwrites if the key already exists. - -**Protocol (RESP2):** - -``` -*3 -$3 -SET -$3 -foo -$3 -bar -``` - -(client sends: array of 3 bulk strings: `["SET", "foo", "bar"]`) - -**Reply:** - -``` -+OK -``` - ---- - -### `GET key` - -* Retrieves the string value stored at the key. -* Returns `nil` if key doesn’t exist. - -**Protocol:** - -``` -*2 -$3 -GET -$3 -foo -``` - -**Reply:** - -``` -$3 -bar -``` - -(or `$-1` for nil) - ---- - -### `DEL key [key ...]` - -* Removes one or more keys. -* Returns number of keys actually removed. - -**Protocol:** - -``` -*2 -$3 -DEL -$3 -foo -``` - -**Reply:** - -``` -:1 -``` - -(integer reply = number of deleted keys) - ---- - -### `EXISTS key [key ...]` - -* Checks if one or more keys exist. -* Returns count of existing keys. - -**Protocol:** - -``` -*2 -$6 -EXISTS -$3 -foo -``` - -**Reply:** - -``` -:1 -``` - ---- - -### `KEYS pattern` - -* Returns all keys matching a glob-style pattern. - ⚠️ Not efficient in production (O(N)), better to use `SCAN`. - -**Protocol:** - -``` -*2 -$4 -KEYS -$1 -* -``` - -**Reply:** - -``` -*2 -$3 -foo -$3 -bar -``` - -(array of bulk strings with key names) - ---- - -# 2. Iteration Commands (`SCAN` family) - -### `SCAN cursor [MATCH pattern] [COUNT n]` - -* Iterates the keyspace incrementally. -* Client keeps sending back the cursor from previous call until it returns `0`. - -**Protocol example:** - -``` -*2 -$4 -SCAN -$1 -0 -``` - -**Reply:** - -``` -*2 -$1 -0 -*2 -$3 -foo -$3 -bar -``` - -Explanation: - -* First element = new cursor (`"0"` means iteration finished). -* Second element = array of keys returned in this batch. - ---- - -### `HSCAN key cursor [MATCH pattern] [COUNT n]` - -* Like `SCAN`, but iterates fields of a hash. - -**Protocol:** - -``` -*3 -$5 -HSCAN -$3 -myh -$1 -0 -``` - -**Reply:** - -``` -*2 -$1 -0 -*4 -$5 -field -$5 -value -$5 -age -$2 -42 -``` - -(Array of alternating field/value pairs) - ---- - -### `SSCAN key cursor [MATCH pattern] [COUNT n]` - -* Iterates members of a set. - -Protocol and reply structure same as SCAN. - ---- - -### `ZSCAN key cursor [MATCH pattern] [COUNT n]` - -* Iterates members of a sorted set with scores. -* Returns alternating `member`, `score`. - ---- - -# Quick Comparison - -| Command | Purpose | Return Type | -| -------- | ----------------------------- | --------------------- | -| `SET` | Store a string value | Simple string `+OK` | -| `GET` | Retrieve a string value | Bulk string / nil | -| `DEL` | Delete keys | Integer (count) | -| `EXISTS` | Check existence | Integer (count) | -| `KEYS` | List all matching keys (slow) | Array of bulk strings | -| `SCAN` | Iterate over keys (safe) | `[cursor, array]` | -| `HSCAN` | Iterate over hash fields | `[cursor, array]` | -| `SSCAN` | Iterate over set members | `[cursor, array]` | -| `ZSCAN` | Iterate over sorted set | `[cursor, array]` | - -## \ No newline at end of file diff --git a/herodb/instructions/redis_hset_functions.md b/herodb/instructions/redis_hset_functions.md deleted file mode 100644 index e91bd7e..0000000 --- a/herodb/instructions/redis_hset_functions.md +++ /dev/null @@ -1,307 +0,0 @@ - -# 🔑 Redis `HSET` and Related Hash Commands - -## 1. `HSET` - -* **Purpose**: Set the value of one or more fields in a hash. -* **Syntax**: - - ```bash - HSET key field value [field value ...] - ``` -* **Return**: - - * Integer: number of fields that were newly added. -* **RESP Protocol**: - - ``` - *4 - $4 - HSET - $3 - key - $5 - field - $5 - value - ``` - - (If multiple field-value pairs: `*6`, `*8`, etc.) - ---- - -## 2. `HSETNX` - -* **Purpose**: Set the value of a hash field only if it does **not** exist. -* **Syntax**: - - ```bash - HSETNX key field value - ``` -* **Return**: - - * `1` if field was set. - * `0` if field already exists. -* **RESP Protocol**: - - ``` - *4 - $6 - HSETNX - $3 - key - $5 - field - $5 - value - ``` - ---- - -## 3. `HGET` - -* **Purpose**: Get the value of a hash field. -* **Syntax**: - - ```bash - HGET key field - ``` -* **Return**: - - * Bulk string (value) or `nil` if field does not exist. -* **RESP Protocol**: - - ``` - *3 - $4 - HGET - $3 - key - $5 - field - ``` - ---- - -## 4. `HGETALL` - -* **Purpose**: Get all fields and values in a hash. -* **Syntax**: - - ```bash - HGETALL key - ``` -* **Return**: - - * Array of `[field1, value1, field2, value2, ...]`. -* **RESP Protocol**: - - ``` - *2 - $7 - HGETALL - $3 - key - ``` - ---- - -## 5. `HMSET` (⚠️ Deprecated, use `HSET`) - -* **Purpose**: Set multiple field-value pairs. -* **Syntax**: - - ```bash - HMSET key field value [field value ...] - ``` -* **Return**: - - * Always `OK`. -* **RESP Protocol**: - - ``` - *6 - $5 - HMSET - $3 - key - $5 - field - $5 - value - $5 - field2 - $5 - value2 - ``` - ---- - -## 6. `HMGET` - -* **Purpose**: Get values of multiple fields. -* **Syntax**: - - ```bash - HMGET key field [field ...] - ``` -* **Return**: - - * Array of values (bulk strings or nils). -* **RESP Protocol**: - - ``` - *4 - $5 - HMGET - $3 - key - $5 - field1 - $5 - field2 - ``` - ---- - -## 7. `HDEL` - -* **Purpose**: Delete one or more fields from a hash. -* **Syntax**: - - ```bash - HDEL key field [field ...] - ``` -* **Return**: - - * Integer: number of fields removed. -* **RESP Protocol**: - - ``` - *3 - $4 - HDEL - $3 - key - $5 - field - ``` - ---- - -## 8. `HEXISTS` - -* **Purpose**: Check if a field exists. -* **Syntax**: - - ```bash - HEXISTS key field - ``` -* **Return**: - - * `1` if exists, `0` if not. -* **RESP Protocol**: - - ``` - *3 - $7 - HEXISTS - $3 - key - $5 - field - ``` - ---- - -## 9. `HKEYS` - -* **Purpose**: Get all field names in a hash. -* **Syntax**: - - ```bash - HKEYS key - ``` -* **Return**: - - * Array of field names. -* **RESP Protocol**: - - ``` - *2 - $5 - HKEYS - $3 - key - ``` - ---- - -## 10. `HVALS` - -* **Purpose**: Get all values in a hash. -* **Syntax**: - - ```bash - HVALS key - ``` -* **Return**: - - * Array of values. -* **RESP Protocol**: - - ``` - *2 - $5 - HVALS - $3 - key - ``` - ---- - -## 11. `HLEN` - -* **Purpose**: Get number of fields in a hash. -* **Syntax**: - - ```bash - HLEN key - ``` -* **Return**: - - * Integer: number of fields. -* **RESP Protocol**: - - ``` - *2 - $4 - HLEN - $3 - key - ``` - - - -## 12. `HSCAN` - -* **Purpose**: Iterate fields/values of a hash (cursor-based scan). -* **Syntax**: - - ```bash - HSCAN key cursor [MATCH pattern] [COUNT count] - ``` -* **Return**: - - * Array: `[new-cursor, [field1, value1, ...]]` -* **RESP Protocol**: - - ``` - *3 - $5 - HSCAN - $3 - key - $1 - 0 - ``` diff --git a/herodb/instructions/redis_lists.md b/herodb/instructions/redis_lists.md deleted file mode 100644 index d42e117..0000000 --- a/herodb/instructions/redis_lists.md +++ /dev/null @@ -1,259 +0,0 @@ - -# 1) Data model & basics - -* A **queue** is a List at key `queue:`. -* Common patterns: - - * **Producer**: `LPUSH queue item` (or `RPUSH`) - * **Consumer (non-blocking)**: `RPOP queue` (or `LPOP`) - * **Consumer (blocking)**: `BRPOP queue timeout` (or `BLPOP`) -* If a key doesn’t exist, it’s treated as an **empty list**; push **creates** the list; when the **last element is popped, the key is deleted**. ([Redis][1]) - ---- - -# 2) Commands to implement (queues via Lists) - -## LPUSH / RPUSH - -Prepend/append one or more elements. Create the list if it doesn’t exist. -**Return**: Integer = new length of the list. - -**Syntax** - -``` -LPUSH key element [element ...] -RPUSH key element [element ...] -``` - -**RESP (example)** - -``` -*3\r\n$5\r\nLPUSH\r\n$5\r\nqueue\r\n$5\r\njob-1\r\n -:1\r\n -``` - -Refs: semantics & multi-arg ordering. ([Redis][1]) - -### LPUSHX / RPUSHX (optional but useful) - -Like LPUSH/RPUSH, **but only if the list exists**. -**Return**: Integer = new length (0 if key didn’t exist). - -``` -LPUSHX key element [element ...] -RPUSHX key element [element ...] -``` - -Refs: command index. ([Redis][2]) - ---- - -## LPOP / RPOP - -Remove & return one (default) or **up to COUNT** elements since Redis 6.2. -If the list is empty or missing, **Null** is returned (Null Bulk or Null Array if COUNT>1). -**Return**: - -* No COUNT: Bulk String or Null Bulk. -* With COUNT: Array of Bulk Strings (possibly empty) or Null Array if key missing. - -**Syntax** - -``` -LPOP key [count] -RPOP key [count] -``` - -**RESP (no COUNT)** - -``` -*2\r\n$4\r\nRPOP\r\n$5\r\nqueue\r\n -$5\r\njob-1\r\n # or $-1\r\n if empty -``` - -**RESP (COUNT=2)** - -``` -*3\r\n$4\r\nLPOP\r\n$5\r\nqueue\r\n$1\r\n2\r\n -*2\r\n$5\r\njob-2\r\n$5\r\njob-3\r\n # or *-1\r\n if key missing -``` - -Refs: LPOP w/ COUNT; general pop semantics. ([Redis][3]) - ---- - -## BLPOP / BRPOP (blocking consumers) - -Block until an element is available in any of the given lists or until `timeout` (seconds, **double**, `0` = forever). -**Return** on success: **Array \[key, element]**. -**Return** on timeout: **Null Array**. - -**Syntax** - -``` -BLPOP key [key ...] timeout -BRPOP key [key ...] timeout -``` - -**RESP** - -``` -*3\r\n$5\r\nBRPOP\r\n$5\r\nqueue\r\n$1\r\n0\r\n # block forever - -# Success reply -*2\r\n$5\r\nqueue\r\n$5\r\njob-4\r\n - -# Timeout reply -*-1\r\n -``` - -**Implementation notes** - -* If any listed key is non-empty at call time, reply **immediately** from the first non-empty key **by the command’s key order**. -* Otherwise, put the client into a **blocked state** (register per-key waiters). On any `LPUSH/RPUSH` to those keys, **wake the earliest waiter** and serve it atomically. -* If timeout expires, return **Null Array** and clear the blocked state. - Refs: timeout semantics and return shape. ([Redis][4]) - ---- - -## LMOVE / BLMOVE (atomic move; replaces RPOPLPUSH/BRPOPLPUSH) - -Atomically **pop from one side** of `source` and **push to one side** of `destination`. - -* Use for **reliable queues** (move to a *processing* list). -* `BLMOVE` blocks like `BLPOP` when `source` is empty. - -**Syntax** - -``` -LMOVE source destination LEFT|RIGHT LEFT|RIGHT -BLMOVE source destination LEFT|RIGHT LEFT|RIGHT timeout -``` - -**Return**: Bulk String element moved, or Null if `source` empty (LMOVE); `BLMOVE` blocks/Null on timeout. - -**RESP (LMOVE RIGHT->LEFT)** - -``` -*5\r\n$5\r\nLMOVE\r\n$6\r\nsource\r\n$3\r\ndst\r\n$5\r\nRIGHT\r\n$4\r\nLEFT\r\n -$5\r\njob-5\r\n -``` - -**Notes** - -* Prefer `LMOVE/BLMOVE` over deprecated `RPOPLPUSH/BRPOPLPUSH`. -* Pattern: consumer `LMOVE queue processing RIGHT LEFT` → work → `LREM processing 1 ` to ACK; a reaper can requeue stale items. - Refs: LMOVE/BLMOVE behavior and reliable-queue pattern; deprecation of RPOPLPUSH. ([Redis][5]) - -*(Compat: you can still implement `RPOPLPUSH source dest` and `BRPOPLPUSH source dest timeout`, but mark them deprecated and map to LMOVE/BLMOVE.)* ([Redis][6]) - ---- - -## LLEN (length) - -Useful for metrics/backpressure. - -``` -LLEN key -``` - -**RESP** - -``` -*2\r\n$4\r\nLLEN\r\n$5\r\nqueue\r\n -:3\r\n -``` - -Refs: list overview mentioning LLEN. ([Redis][7]) - ---- - -## LREM (ack for “reliable” processing) - -Remove occurrences of `element` from the list (head→tail scan). -Use `count=1` to ACK a single processed item from `processing`. - -``` -LREM key count element -``` - -**RESP** - -``` -*4\r\n$4\r\nLREM\r\n$9\r\nprocessing\r\n$1\r\n1\r\n$5\r\njob-5\r\n -:1\r\n -``` - -Refs: reliable pattern mentions LREM to ACK. ([Redis][5]) - ---- - -## LTRIM (bounded queues / retention) - -Keep only `[start, stop]` range; everything else is dropped. -Use to cap queue length after pushes. - -``` -LTRIM key start stop -``` - -**RESP** - -``` -*4\r\n$5\r\nLTRIM\r\n$5\r\nqueue\r\n$2\r\n0\r\n$3\r\n999\r\n -+OK\r\n -``` - -Refs: list overview includes LTRIM for retention. ([Redis][7]) - ---- - -## LRANGE / LINDEX (debugging / peeking) - -* `LRANGE key start stop` → Array of elements (non-destructive). -* `LINDEX key index` → one element or Null. - -These aren’t required for queue semantics, but handy. ([Redis][7]) - ---- - -# 3) Errors & types - -* Wrong type: `-WRONGTYPE Operation against a key holding the wrong kind of value\r\n` -* Non-existing key: - - * Push: creates the list (returns new length). - * Pop (non-blocking): returns **Null**. - * Blocking pop: **Null Array** on timeout. ([Redis][1]) - ---- - -# 4) Blocking engine (implementation sketch) - -1. **Call time**: scan keys in user order. If a non-empty list is found, pop & reply immediately. -2. **Otherwise**: register the client as **blocked** on those keys with `deadline = now + timeout` (or infinite). -3. **On push to any key**: if waiters exist, **wake one** (FIFO) and serve its pop **atomically** with the push result. -4. **On timer**: for each blocked client whose deadline passed, reply `Null Array` and clear state. -5. **Connection close**: remove from any wait queues. - -Refs for timeout/block semantics. ([Redis][4]) - ---- - -# 5) Reliable queue pattern (recommended) - -* **Consume**: `LMOVE queue processing RIGHT LEFT` (or `BLMOVE ... 0`). -* **Process** the job. -* **ACK**: `LREM processing 1 ` when done. -* **Reaper**: auxiliary task that detects stale jobs (e.g., track job IDs + timestamps in a ZSET) and requeues them. (Lists don’t include timestamps; pairing with a ZSET is standard practice.) - Refs: LMOVE doc’s pattern. ([Redis][5]) - ---- - -# 6) Minimal test matrix - -* Push/pop happy path (both ends), with/without COUNT. -* Blocking pop: immediate availability, block + timeout, wake on push, multiple keys order, FIFO across multiple waiters. -* LMOVE/BLMOVE: RIGHT→LEFT pipeline, block + wake, cross-list atomicity, ACK via LREM. -* Type errors and key deletion on last pop. - diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..e71fdf5 --- /dev/null +++ b/src/main.rs @@ -0,0 +1 @@ +fn main() {} \ No newline at end of file