From 9054737e843a63d6b45e3d14ca71764a04415e1e Mon Sep 17 00:00:00 2001 From: despiegk Date: Fri, 22 Aug 2025 17:09:08 +0200 Subject: [PATCH] ... --- Cargo.lock | 123 +++- herodb/Cargo.toml | 3 +- herodb/README.md | 85 +++ herodb/docs/cmds.md | 307 +++------ herodb/specs/backgroundinfo/tantivy.md | 0 herodb/src/crypto.rs | 1 + herodb/src/lib.rs | 2 + herodb/src/main.rs | 10 +- herodb/src/options.rs | 12 +- herodb/src/storage/mod.rs | 157 +++++ herodb/src/storage_sled/mod.rs | 837 +++++++++++++++++++++++++ herodb/src/storage_trait.rs | 57 ++ 12 files changed, 1373 insertions(+), 221 deletions(-) create mode 100644 herodb/specs/backgroundinfo/tantivy.md create mode 100644 herodb/src/storage_sled/mod.rs create mode 100644 herodb/src/storage_trait.rs diff --git a/Cargo.lock b/Cargo.lock index 15bf240..fa0053c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -206,6 +206,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.9.2" @@ -358,6 +364,30 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.6" @@ -406,7 +436,7 @@ dependencies = [ "hashbrown", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.11", ] [[package]] @@ -533,6 +563,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures" version = "0.3.31" @@ -622,6 +662,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -682,6 +731,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "sled", "thiserror", "tokio", ] @@ -732,7 +782,7 @@ dependencies = [ "intl-memoizer", "lazy_static", "log", - "parking_lot", + "parking_lot 0.12.4", "rust-embed", "thiserror", "unic-langid", @@ -889,6 +939,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "intl-memoizer" version = "0.5.3" @@ -914,7 +973,7 @@ version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" dependencies = [ - "bitflags", + "bitflags 2.9.2", "cfg-if", "libc", ] @@ -1040,6 +1099,17 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.4" @@ -1047,7 +1117,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.11", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -1058,7 +1142,7 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.17", "smallvec", "windows-targets 0.52.6", ] @@ -1252,13 +1336,22 @@ dependencies = [ "url", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags", + "bitflags 2.9.2", ] [[package]] @@ -1466,6 +1559,22 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -1599,7 +1708,7 @@ dependencies = [ "io-uring", "libc", "mio", - "parking_lot", + "parking_lot 0.12.4", "pin-project-lite", "signal-hook-registry", "slab", diff --git a/herodb/Cargo.toml b/herodb/Cargo.toml index b71eb22..7e952b6 100644 --- a/herodb/Cargo.toml +++ b/herodb/Cargo.toml @@ -12,10 +12,11 @@ tokio = { version = "1.23.0", features = ["full"] } clap = { version = "4.5.20", features = ["derive"] } byteorder = "1.4.3" futures = "0.3" +sled = "0.34" redb = "2.1.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -bincode = "1.3.3" +bincode = "1.3" chacha20poly1305 = "0.10.1" rand = "0.8" sha2 = "0.10" diff --git a/herodb/README.md b/herodb/README.md index e69de29..46d4c57 100644 --- a/herodb/README.md +++ b/herodb/README.md @@ -0,0 +1,85 @@ +# HeroDB + +HeroDB is a Redis-compatible database built with Rust, offering a flexible and secure storage solution. It supports two primary storage backends: `redb` (default) and `sled`, both with full encryption capabilities. HeroDB aims to provide a robust and performant key-value store with advanced features like data-at-rest encryption, hash operations, list operations, and cursor-based scanning. + +## Purpose + +The main purpose of HeroDB is to offer a lightweight, embeddable, and Redis-compatible database that prioritizes data security through transparent encryption. It's designed for applications that require fast, reliable data storage with the option for strong cryptographic protection, without the overhead of a full-fledged Redis server. + +## Features + +- **Redis Compatibility**: Supports a subset of Redis commands over RESP (Redis Serialization Protocol) via TCP. +- **Dual Backend Support**: + - `redb` (default): Optimized for concurrent access and high-throughput scenarios. + - `sled`: A lock-free, log-structured database, excellent for specific workloads. +- **Data-at-Rest Encryption**: Transparent encryption for both backends using the `age` encryption library. +- **Key-Value Operations**: Full support for basic string, hash, and list operations. +- **Expiration**: Time-to-live (TTL) functionality for keys. +- **Scanning**: Cursor-based iteration for keys and hash fields (`SCAN`, `HSCAN`). +- **AGE Cryptography Commands**: HeroDB-specific extensions for cryptographic operations. + +## Quick Start + +### Building HeroDB + +To build HeroDB, navigate to the project root and run: + +```bash +cargo build --release +``` + +### Running HeroDB + +You can start HeroDB with different backends and encryption options: + +#### Default `redb` Backend + +```bash +./target/release/herodb --dir /tmp/herodb_redb --port 6379 +``` + +#### `sled` Backend + +```bash +./target/release/herodb --dir /tmp/herodb_sled --port 6379 --sled +``` + +#### `redb` with Encryption + +```bash +./target/release/herodb --dir /tmp/herodb_encrypted --port 6379 --encrypt --key mysecretkey +``` + +#### `sled` with Encryption + +```bash +./target/release/herodb --dir /tmp/herodb_sled_encrypted --port 6379 --sled --encrypt --key mysecretkey +``` + +## Usage with Redis Clients + +HeroDB can be interacted with using any standard Redis client, such as `redis-cli`, `redis-py` (Python), or `ioredis` (Node.js). + +### Example with `redis-cli` + +```bash +redis-cli -p 6379 SET mykey "Hello from HeroDB!" +redis-cli -p 6379 GET mykey +# → "Hello from HeroDB!" + +redis-cli -p 6379 HSET user:1 name "Alice" age "30" +redis-cli -p 6379 HGET user:1 name +# → "Alice" + +redis-cli -p 6379 SCAN 0 MATCH user:* COUNT 10 +# → 1) "0" +# 2) 1) "user:1" +``` + +## Documentation + +For more detailed information on commands, features, and advanced usage, please refer to the documentation: + +- [Basics](docs/basics.md) +- [Supported Commands](docs/cmds.md) +- [AGE Cryptography](docs/age.md) \ No newline at end of file diff --git a/herodb/docs/cmds.md b/herodb/docs/cmds.md index 140638a..fa85ff4 100644 --- a/herodb/docs/cmds.md +++ b/herodb/docs/cmds.md @@ -1,227 +1,116 @@ -# HeroDB Redis Protocol Support: Commands & Client Usage +## Backend Support -HeroDB is a Redis-compatible database built using the `redb` database backend. +HeroDB supports two storage backends, both with full encryption support: -It supports a subset of Redis commands over the standard RESP (Redis Serialization Protocol) via TCP, allowing you to interact with it using standard Redis clients like `redis-cli`, Python's `redis-py`, Node.js's `ioredis`, etc. +- **redb** (default): Full-featured, optimized for production use +- **sled**: Alternative embedded database with encryption support -This document provides: -- A list of all currently supported Redis commands. -- Example usage with standard Redis clients. -- Bash and Rust test-inspired usage examples. - -## Quick Start - -Assuming the server is running on localhost at port `$PORT`: +### Starting HeroDB with Different Backends ```bash -# Build HeroDB -cargo build --release +# Use default redb backend +./target/release/herodb --dir /tmp/herodb_redb --port 6379 -# Start HeroDB server -./target/release/herodb --dir /tmp/herodb_data --port 6381 --debug +# Use sled backend +./target/release/herodb --dir /tmp/herodb_sled --port 6379 --sled + +# Use redb with encryption +./target/release/herodb --dir /tmp/herodb_encrypted --port 6379 --encrypt --key mysecretkey + +# Use sled with encryption +./target/release/herodb --dir /tmp/herodb_sled_encrypted --port 6379 --sled --encrypt --key mysecretkey ``` -## Using Standard Redis Clients +### Command Support by Backend -### With `redis-cli` +Command Category | redb | sled | Notes | +|-----------------|------|------|-------| +**Strings** | | | | +SET | ✅ | ✅ | Full support | +GET | ✅ | ✅ | Full support | +DEL | ✅ | ✅ | Full support | +EXISTS | ✅ | ✅ | Full support | +INCR/DECR | ✅ | ✅ | Full support | +MGET/MSET | ✅ | ✅ | Full support | +**Hashes** | | | | +HSET | ✅ | ✅ | Full support | +HGET | ✅ | ✅ | Full support | +HGETALL | ✅ | ✅ | Full support | +HDEL | ✅ | ✅ | Full support | +HEXISTS | ✅ | ✅ | Full support | +HKEYS | ✅ | ✅ | Full support | +HVALS | ✅ | ✅ | Full support | +HLEN | ✅ | ✅ | Full support | +HMGET | ✅ | ✅ | Full support | +HSETNX | ✅ | ✅ | Full support | +HINCRBY/HINCRBYFLOAT | ✅ | ✅ | Full support | +HSCAN | ✅ | ✅ | Full support with pattern matching | +**Lists** | | | | +LPUSH/RPUSH | ✅ | ✅ | Full support | +LPOP/RPOP | ✅ | ✅ | Full support | +LLEN | ✅ | ✅ | Full support | +LRANGE | ✅ | ✅ | Full support | +LINDEX | ✅ | ✅ | Full support | +LTRIM | ✅ | ✅ | Full support | +LREM | ✅ | ✅ | Full support | +BLPOP/BRPOP | ✅ | ❌ | Blocking operations not in sled | +**Expiration** | | | | +EXPIRE | ✅ | ✅ | Full support in both | +TTL | ✅ | ✅ | Full support in both | +PERSIST | ✅ | ✅ | Full support in both | +SETEX/PSETEX | ✅ | ✅ | Full support in both | +EXPIREAT/PEXPIREAT | ✅ | ✅ | Full support in both | +**Scanning** | | | | +KEYS | ✅ | ✅ | Full support with patterns | +SCAN | ✅ | ✅ | Full cursor-based iteration | +HSCAN | ✅ | ✅ | Full cursor-based iteration | +**Transactions** | | | | +MULTI/EXEC/DISCARD | ✅ | ❌ | Only supported in redb | +**Encryption** | | | | +Data-at-rest encryption | ✅ | ✅ | Both support [age](age.tech) encryption | +AGE commands | ✅ | ✅ | Both support AGE crypto commands | + +### Performance Considerations + +- **redb**: Optimized for concurrent access, better for high-throughput scenarios +- **sled**: Lock-free architecture, excellent for specific workloads + +### Encryption Features + +Both backends support: +- Transparent data-at-rest encryption using the `age` encryption library +- Per-database encryption (databases >= 10 are encrypted when `--encrypt` flag is used) +- Secure key derivation using the master key + +### Backend Selection Examples ```bash -redis-cli -p 6381 SET mykey "hello" -redis-cli -p 6381 GET mykey +# Example: Testing both backends +redis-cli -p 6379 SET mykey "redb value" +redis-cli -p 6381 SET mykey "sled value" + +# Example: Using encryption with both +./target/release/herodb --port 6379 --encrypt --key secret123 +./target/release/herodb --port 6381 --sled --encrypt --key secret123 + +# Both support the same Redis commands +redis-cli -p 6379 HSET user:1 name "Alice" age "30" +redis-cli -p 6381 HSET user:1 name "Alice" age "30" + +# Both support SCAN operations +redis-cli -p 6379 SCAN 0 MATCH user:* COUNT 10 +redis-cli -p 6381 SCAN 0 MATCH user:* COUNT 10 ``` -### With Python (`redis-py`) +### Migration Between Backends -```python -import redis - -r = redis.Redis(host='localhost', port=6381, db=0) -r.set('mykey', 'hello') -print(r.get('mykey').decode()) -``` - -### With Node.js (`ioredis`) - -```js -const Redis = require("ioredis"); -const redis = new Redis({ port: 6381, host: "localhost" }); - -await redis.set("mykey", "hello"); -const value = await redis.get("mykey"); -console.log(value); // "hello" -``` - -## Supported Redis Commands - -### String Commands - -| Command | Description | Example Usage | -|---------------|------------------------------------------|-------------------------------------------| -| `SET` | Set a key to a string value | `SET name "Alice"` | -| `GET` | Get the value of a key | `GET name` | -| `DEL` | Delete one or more keys | `DEL name age` | -| `INCR` | Increment the integer value of a key | `INCR counter` | -| `DECR` | Decrement the integer value of a key | `DECR counter` | -| `INCRBY` | Increment key by a given integer | `INCRBY counter 5` | -| `DECRBY` | Decrement key by a given integer | `DECRBY counter 3` | -| `EXISTS` | Check if a key exists | `EXISTS name` | -| `TYPE` | Return the type of a key | `TYPE name` | - -### Hash Commands - -| Command | Description | Example Usage | -|---------------|------------------------------------------|-------------------------------------------| -| `HSET` | Set field in hash stored at key | `HSET user:1 name "Alice"` | -| `HGET` | Get value of a field in hash | `HGET user:1 name` | -| `HGETALL` | Get all fields and values in a hash | `HGETALL user:1` | -| `HDEL` | Delete one or more fields from hash | `HDEL user:1 name age` | -| `HEXISTS` | Check if field exists in hash | `HEXISTS user:1 name` | -| `HKEYS` | Get all field names in a hash | `HKEYS user:1` | -| `HVALS` | Get all values in a hash | `HVALS user:1` | -| `HLEN` | Get number of fields in a hash | `HLEN user:1` | -| `HMGET` | Get values of multiple fields | `HMGET user:1 name age` | -| `HSETNX` | Set field only if it does not exist | `HSETNX user:1 email alice@example.com` | - -### List Commands - -| Command | Description | Example Usage | -|---------------|------------------------------------------|-------------------------------------------| -| `LPUSH` | Insert elements at the head of a list | `LPUSH mylist "item1" "item2"` | -| `RPUSH` | Insert elements at the tail of a list | `RPUSH mylist "item3" "item4"` | -| `LPOP` | Remove and return element from head | `LPOP mylist` | -| `RPOP` | Remove and return element from tail | `RPOP mylist` | -| `BLPOP` | Blocking remove from head with timeout | `BLPOP mylist1 mylist2 5` | -| `BRPOP` | Blocking remove from tail with timeout | `BRPOP mylist1 mylist2 5` | -| `LLEN` | Get the length of a list | `LLEN mylist` | -| `LREM` | Remove elements from list | `LREM mylist 2 "item"` | -| `LTRIM` | Trim list to specified range | `LTRIM mylist 0 5` | -| `LINDEX` | Get element by index | `LINDEX mylist 0` | -| `LRANGE` | Get range of elements | `LRANGE mylist 0 -1` | - -### Keys & Scanning - -| Command | Description | Example Usage | -|---------------|------------------------------------------|-------------------------------------------| -| `KEYS` | Find all keys matching a pattern | `KEYS user:*` | -| `SCAN` | Incrementally iterate keys | `SCAN 0 MATCH user:* COUNT 10` | - -### Expiration - -| Command | Description | Example Usage | -|---------------|------------------------------------------|-------------------------------------------| -| `EXPIRE` | Set a key's time to live in seconds | `EXPIRE tempkey 60` | -| `TTL` | Get the time to live for a key | `TTL tempkey` | -| `PERSIST` | Remove the expiration from a key | `PERSIST tempkey` | - -### Transactions - -| Command | Description | Example Usage | -|---------------|------------------------------------------|-------------------------------------------| -| `MULTI` | Start a transaction block | `MULTI` | -| `EXEC` | Execute all commands in a transaction | `EXEC` | -| `DISCARD` | Discard all commands in a transaction | `DISCARD` | - -### Configuration - -| Command | Description | Example Usage | -|---------------|------------------------------------------|-------------------------------------------| -| `CONFIG GET` | Get configuration parameters | `CONFIG GET dir` | -| `CONFIG SET` | Set configuration parameters | `CONFIG SET maxmemory 100mb` | - -### Info & Monitoring - -| Command | Description | Example Usage | -|---------------|------------------------------------------|-------------------------------------------| -| `INFO` | Get information and statistics about server | `INFO` | -| `PING` | Ping the server | `PING` | - -### AGE Cryptography Commands - -| Command | Description | Example Usage | -|--------------------|-----------------------------------------------|-----------------------------------------------| -| `AGE GENENC` | Generate ephemeral encryption keypair | `AGE GENENC` | -| `AGE GENSIGN` | Generate ephemeral signing keypair | `AGE GENSIGN` | -| `AGE ENCRYPT` | Encrypt a message using a public key | `AGE ENCRYPT "msg"` | -| `AGE DECRYPT` | Decrypt a message using a secret key | `AGE DECRYPT ` | -| `AGE SIGN` | Sign a message using a secret key | `AGE SIGN "msg"` | -| `AGE VERIFY` | Verify a signature using a public key | `AGE VERIFY "msg" ` | -| `AGE KEYGEN` | Create and persist a named encryption key | `AGE KEYGEN app1` | -| `AGE SIGNKEYGEN` | Create and persist a named signing key | `AGE SIGNKEYGEN app1` | -| `AGE ENCRYPTNAME` | Encrypt using a named key | `AGE ENCRYPTNAME app1 "msg"` | -| `AGE DECRYPTNAME` | Decrypt using a named key | `AGE DECRYPTNAME app1 ` | -| `AGE SIGNNAME` | Sign using a named key | `AGE SIGNNAME app1 "msg"` | -| `AGE VERIFYNAME` | Verify using a named key | `AGE VERIFYNAME app1 "msg" ` | -| `AGE LIST` | List all persisted named keys | `AGE LIST` | - -> Note: AGE commands are not part of standard Redis. They are HeroDB-specific extensions for cryptographic operations. - -## Example Usage - -### Basic String Operations +To migrate data between backends, use Redis replication or dump/restore: ```bash -redis-cli -p 6381 SET greeting "Hello, HeroDB!" -redis-cli -p 6381 GET greeting -# → "Hello, HeroDB!" +# Export from redb +redis-cli -p 6379 --rdb dump.rdb -redis-cli -p 6381 INCR visits -redis-cli -p 6381 INCR visits -redis-cli -p 6381 GET visits -# → "2" -``` - -### Hash Operations - -```bash -redis-cli -p 6381 HSET user:1000 name "Alice" age "30" city "NYC" -redis-cli -p 6381 HGET user:1000 name -# → "Alice" - -redis-cli -p 6381 HGETALL user:1000 -# → 1) "name" -# 2) "Alice" -# 3) "age" -# 4) "30" -# 5) "city" -# 6) "NYC" -``` - -### Expiration - -```bash -redis-cli -p 6381 SET tempkey "temporary" -redis-cli -p 6381 EXPIRE tempkey 5 -redis-cli -p 6381 TTL tempkey -# → (integer) 4 - -# After 5 seconds: -redis-cli -p 6381 GET tempkey -# → (nil) -``` - -### Transactions - -```bash -redis-cli -p 6381 MULTI -redis-cli -p 6381 SET txkey1 "value1" -redis-cli -p 6381 SET txkey2 "value2" -redis-cli -p 6381 INCR counter -redis-cli -p 6381 EXEC -# → 1) OK -# 2) OK -# 3) (integer) 3 -``` - -### Scanning Keys - -```bash -redis-cli -p 6381 SET scankey1 "val1" -redis-cli -p 6381 SET scankey2 "val2" -redis-cli -p 6381 HSET scanhash field1 "val1" - -redis-cli -p 6381 SCAN 0 MATCH scankey* -# → 1) "0" -# 2) 1) "scankey1" -# 2) "scankey2" -``` +# Import to sled +redis-cli -p 6381 --pipe < dump.rdb +``` \ No newline at end of file diff --git a/herodb/specs/backgroundinfo/tantivy.md b/herodb/specs/backgroundinfo/tantivy.md new file mode 100644 index 0000000..e69de29 diff --git a/herodb/src/crypto.rs b/herodb/src/crypto.rs index e43317b..48a9f8c 100644 --- a/herodb/src/crypto.rs +++ b/herodb/src/crypto.rs @@ -23,6 +23,7 @@ impl From for crate::error::DBError { } /// Super-simple factory: new(secret) + encrypt(bytes) + decrypt(bytes) +#[derive(Clone)] pub struct CryptoFactory { key: chacha20poly1305::Key, } diff --git a/herodb/src/lib.rs b/herodb/src/lib.rs index a66b56e..31e69a8 100644 --- a/herodb/src/lib.rs +++ b/herodb/src/lib.rs @@ -6,3 +6,5 @@ pub mod options; pub mod protocol; pub mod server; pub mod storage; +pub mod storage_trait; // Add this +pub mod storage_sled; // Add this diff --git a/herodb/src/main.rs b/herodb/src/main.rs index 1e3d2c6..48f74dd 100644 --- a/herodb/src/main.rs +++ b/herodb/src/main.rs @@ -30,6 +30,10 @@ struct Args { /// Encrypt the database #[arg(long)] encrypt: bool, + + /// Use the sled backend + #[arg(long)] + sled: bool, } #[tokio::main] @@ -47,10 +51,14 @@ async fn main() { // new DB option let option = herodb::options::DBOption { dir: args.dir, - port, debug: args.debug, encryption_key: args.encryption_key, encrypt: args.encrypt, + backend: if args.sled { + herodb::options::BackendType::Sled + } else { + herodb::options::BackendType::Redb + }, }; // new server diff --git a/herodb/src/options.rs b/herodb/src/options.rs index 26da765..284c893 100644 --- a/herodb/src/options.rs +++ b/herodb/src/options.rs @@ -1,8 +1,14 @@ -#[derive(Clone)] +#[derive(Debug, Clone)] +pub enum BackendType { + Redb, + Sled, +} + +#[derive(Debug, Clone)] pub struct DBOption { pub dir: String, - pub port: u16, pub debug: bool, pub encrypt: bool, - pub encryption_key: Option, // Master encryption key + pub encryption_key: Option, + pub backend: BackendType, } diff --git a/herodb/src/storage/mod.rs b/herodb/src/storage/mod.rs index 7c33028..58dc5bf 100644 --- a/herodb/src/storage/mod.rs +++ b/herodb/src/storage/mod.rs @@ -1,5 +1,6 @@ use std::{ path::Path, + sync::Arc, time::{SystemTime, UNIX_EPOCH}, }; @@ -123,4 +124,160 @@ impl Storage { Ok(data.to_vec()) } } +} + +use crate::storage_trait::StorageBackend; + +impl StorageBackend for Storage { + fn get(&self, key: &str) -> Result, DBError> { + self.get(key) + } + + fn set(&self, key: String, value: String) -> Result<(), DBError> { + self.set(key, value) + } + + fn setx(&self, key: String, value: String, expire_ms: u128) -> Result<(), DBError> { + self.setx(key, value, expire_ms) + } + + fn del(&self, key: String) -> Result<(), DBError> { + self.del(key) + } + + fn exists(&self, key: &str) -> Result { + self.exists(key) + } + + fn keys(&self, pattern: &str) -> Result, DBError> { + self.keys(pattern) + } + + fn dbsize(&self) -> Result { + self.dbsize() + } + + fn flushdb(&self) -> Result<(), DBError> { + self.flushdb() + } + + fn get_key_type(&self, key: &str) -> Result, DBError> { + self.get_key_type(key) + } + + fn scan(&self, cursor: u64, pattern: Option<&str>, count: Option) -> Result<(u64, Vec<(String, String)>), DBError> { + self.scan(cursor, pattern, count) + } + + fn hscan(&self, key: &str, cursor: u64, pattern: Option<&str>, count: Option) -> Result<(u64, Vec<(String, String)>), DBError> { + self.hscan(key, cursor, pattern, count) + } + + fn hset(&self, key: &str, pairs: Vec<(String, String)>) -> Result { + self.hset(key, pairs) + } + + fn hget(&self, key: &str, field: &str) -> Result, DBError> { + self.hget(key, field) + } + + fn hgetall(&self, key: &str) -> Result, DBError> { + self.hgetall(key) + } + + fn hdel(&self, key: &str, fields: Vec) -> Result { + self.hdel(key, fields) + } + + fn hexists(&self, key: &str, field: &str) -> Result { + self.hexists(key, field) + } + + fn hkeys(&self, key: &str) -> Result, DBError> { + self.hkeys(key) + } + + fn hvals(&self, key: &str) -> Result, DBError> { + self.hvals(key) + } + + fn hlen(&self, key: &str) -> Result { + self.hlen(key) + } + + fn hmget(&self, key: &str, fields: Vec) -> Result>, DBError> { + self.hmget(key, fields) + } + + fn hsetnx(&self, key: &str, field: &str, value: &str) -> Result { + self.hsetnx(key, field, value) + } + + fn lpush(&self, key: &str, elements: Vec) -> Result { + self.lpush(key, elements) + } + + fn rpush(&self, key: &str, elements: Vec) -> Result { + self.rpush(key, elements) + } + + fn lpop(&self, key: &str, count: u64) -> Result, DBError> { + self.lpop(key, count) + } + + fn rpop(&self, key: &str, count: u64) -> Result, DBError> { + self.rpop(key, count) + } + + fn llen(&self, key: &str) -> Result { + self.llen(key) + } + + fn lindex(&self, key: &str, index: i64) -> Result, DBError> { + self.lindex(key, index) + } + + fn lrange(&self, key: &str, start: i64, stop: i64) -> Result, DBError> { + self.lrange(key, start, stop) + } + + fn ltrim(&self, key: &str, start: i64, stop: i64) -> Result<(), DBError> { + self.ltrim(key, start, stop) + } + + fn lrem(&self, key: &str, count: i64, element: &str) -> Result { + self.lrem(key, count, element) + } + + fn ttl(&self, key: &str) -> Result { + self.ttl(key) + } + + fn expire_seconds(&self, key: &str, secs: u64) -> Result { + self.expire_seconds(key, secs) + } + + fn pexpire_millis(&self, key: &str, ms: u128) -> Result { + self.pexpire_millis(key, ms) + } + + fn persist(&self, key: &str) -> Result { + self.persist(key) + } + + fn expire_at_seconds(&self, key: &str, ts_secs: i64) -> Result { + self.expire_at_seconds(key, ts_secs) + } + + fn pexpire_at_millis(&self, key: &str, ts_ms: i64) -> Result { + self.pexpire_at_millis(key, ts_ms) + } + + fn is_encrypted(&self) -> bool { + self.is_encrypted() + } + + fn clone_arc(&self) -> Arc { + unimplemented!("Storage cloning not yet implemented for redb backend") + } } \ No newline at end of file diff --git a/herodb/src/storage_sled/mod.rs b/herodb/src/storage_sled/mod.rs new file mode 100644 index 0000000..d6fd862 --- /dev/null +++ b/herodb/src/storage_sled/mod.rs @@ -0,0 +1,837 @@ +// src/storage_sled/mod.rs +use std::path::Path; +use std::sync::Arc; +use std::collections::HashMap; +use std::time::{SystemTime, UNIX_EPOCH}; +use serde::{Deserialize, Serialize}; +use crate::error::DBError; +use crate::storage_trait::StorageBackend; +use crate::crypto::CryptoFactory; + +#[derive(Serialize, Deserialize, Debug, Clone)] +enum ValueType { + String(String), + Hash(HashMap), + List(Vec), +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +struct StorageValue { + value: ValueType, + expires_at: Option, // milliseconds since epoch +} + +pub struct SledStorage { + db: sled::Db, + types: sled::Tree, + crypto: Option, +} + +impl SledStorage { + pub fn new(path: impl AsRef, should_encrypt: bool, master_key: Option<&str>) -> Result { + let db = sled::open(path).map_err(|e| DBError(format!("Failed to open sled: {}", e)))?; + let types = db.open_tree("types").map_err(|e| DBError(format!("Failed to open types tree: {}", e)))?; + + // Check if database was previously encrypted + let encrypted_tree = db.open_tree("encrypted").map_err(|e| DBError(e.to_string()))?; + let was_encrypted = encrypted_tree.get("encrypted") + .map_err(|e| DBError(e.to_string()))? + .map(|v| v[0] == 1) + .unwrap_or(false); + + let crypto = if should_encrypt || was_encrypted { + if let Some(key) = master_key { + Some(CryptoFactory::new(key.as_bytes())) + } else { + return Err(DBError("Encryption requested but no master key provided".to_string())); + } + } else { + None + }; + + // Mark database as encrypted if enabling encryption + if should_encrypt && !was_encrypted { + encrypted_tree.insert("encrypted", &[1u8]) + .map_err(|e| DBError(e.to_string()))?; + encrypted_tree.flush().map_err(|e| DBError(e.to_string()))?; + } + + Ok(SledStorage { db, types, crypto }) + } + + fn now_millis() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + } + + fn encrypt_if_needed(&self, data: &[u8]) -> Result, DBError> { + if let Some(crypto) = &self.crypto { + Ok(crypto.encrypt(data)) + } else { + Ok(data.to_vec()) + } + } + + fn decrypt_if_needed(&self, data: &[u8]) -> Result, DBError> { + if let Some(crypto) = &self.crypto { + Ok(crypto.decrypt(data)?) + } else { + Ok(data.to_vec()) + } + } + + fn get_storage_value(&self, key: &str) -> Result, DBError> { + match self.db.get(key).map_err(|e| DBError(e.to_string()))? { + Some(encrypted_data) => { + let decrypted = self.decrypt_if_needed(&encrypted_data)?; + let storage_val: StorageValue = bincode::deserialize(&decrypted) + .map_err(|e| DBError(format!("Deserialization error: {}", e)))?; + + // Check expiration + if let Some(expires_at) = storage_val.expires_at { + if Self::now_millis() > expires_at { + // Expired, remove it + self.db.remove(key).map_err(|e| DBError(e.to_string()))?; + self.types.remove(key).map_err(|e| DBError(e.to_string()))?; + return Ok(None); + } + } + + Ok(Some(storage_val)) + } + None => Ok(None) + } + } + + fn set_storage_value(&self, key: &str, storage_val: StorageValue) -> Result<(), DBError> { + let data = bincode::serialize(&storage_val) + .map_err(|e| DBError(format!("Serialization error: {}", e)))?; + let encrypted = self.encrypt_if_needed(&data)?; + self.db.insert(key, encrypted).map_err(|e| DBError(e.to_string()))?; + + // Store type info (unencrypted for efficiency) + let type_str = match &storage_val.value { + ValueType::String(_) => "string", + ValueType::Hash(_) => "hash", + ValueType::List(_) => "list", + }; + self.types.insert(key, type_str.as_bytes()).map_err(|e| DBError(e.to_string()))?; + + Ok(()) + } + + fn glob_match(pattern: &str, text: &str) -> bool { + if pattern == "*" { + return true; + } + + let pattern_chars: Vec = pattern.chars().collect(); + let text_chars: Vec = text.chars().collect(); + + fn match_recursive(pattern: &[char], text: &[char], pi: usize, ti: usize) -> bool { + if pi >= pattern.len() { + return ti >= text.len(); + } + + if ti >= text.len() { + return pattern[pi..].iter().all(|&c| c == '*'); + } + + match pattern[pi] { + '*' => { + for i in ti..=text.len() { + if match_recursive(pattern, text, pi + 1, i) { + return true; + } + } + false + } + '?' => match_recursive(pattern, text, pi + 1, ti + 1), + c => { + if text[ti] == c { + match_recursive(pattern, text, pi + 1, ti + 1) + } else { + false + } + } + } + } + + match_recursive(&pattern_chars, &text_chars, 0, 0) + } +} + +impl StorageBackend for SledStorage { + fn get(&self, key: &str) -> Result, DBError> { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::String(s) => Ok(Some(s)), + _ => Ok(None) + } + None => Ok(None) + } + } + + fn set(&self, key: String, value: String) -> Result<(), DBError> { + let storage_val = StorageValue { + value: ValueType::String(value), + expires_at: None, + }; + self.set_storage_value(&key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(()) + } + + fn setx(&self, key: String, value: String, expire_ms: u128) -> Result<(), DBError> { + let storage_val = StorageValue { + value: ValueType::String(value), + expires_at: Some(Self::now_millis() + expire_ms), + }; + self.set_storage_value(&key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(()) + } + + fn del(&self, key: String) -> Result<(), DBError> { + self.db.remove(&key).map_err(|e| DBError(e.to_string()))?; + self.types.remove(&key).map_err(|e| DBError(e.to_string()))?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(()) + } + + fn exists(&self, key: &str) -> Result { + // Check with expiration + Ok(self.get_storage_value(key)?.is_some()) + } + + fn keys(&self, pattern: &str) -> Result, DBError> { + let mut keys = Vec::new(); + for item in self.types.iter() { + let (key_bytes, _) = item.map_err(|e| DBError(e.to_string()))?; + let key = String::from_utf8_lossy(&key_bytes).to_string(); + + // Check if key is expired + if self.get_storage_value(&key)?.is_some() { + if Self::glob_match(pattern, &key) { + keys.push(key); + } + } + } + Ok(keys) + } + + fn scan(&self, cursor: u64, pattern: Option<&str>, count: Option) -> Result<(u64, Vec<(String, String)>), DBError> { + let mut result = Vec::new(); + let mut current_cursor = 0u64; + let limit = count.unwrap_or(10) as usize; + + for item in self.types.iter() { + if current_cursor >= cursor { + let (key_bytes, type_bytes) = item.map_err(|e| DBError(e.to_string()))?; + let key = String::from_utf8_lossy(&key_bytes).to_string(); + + // Check pattern match + let matches = if let Some(pat) = pattern { + Self::glob_match(pat, &key) + } else { + true + }; + + if matches { + // Check if key is expired and get value + if let Some(storage_val) = self.get_storage_value(&key)? { + let value = match storage_val.value { + ValueType::String(s) => s, + _ => String::from_utf8_lossy(&type_bytes).to_string(), + }; + result.push((key, value)); + + if result.len() >= limit { + current_cursor += 1; + break; + } + } + } + } + current_cursor += 1; + } + + let next_cursor = if result.len() < limit { 0 } else { current_cursor }; + Ok((next_cursor, result)) + } + + fn dbsize(&self) -> Result { + let mut count = 0i64; + for item in self.types.iter() { + let (key_bytes, _) = item.map_err(|e| DBError(e.to_string()))?; + let key = String::from_utf8_lossy(&key_bytes).to_string(); + if self.get_storage_value(&key)?.is_some() { + count += 1; + } + } + Ok(count) + } + + fn flushdb(&self) -> Result<(), DBError> { + self.db.clear().map_err(|e| DBError(e.to_string()))?; + self.types.clear().map_err(|e| DBError(e.to_string()))?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(()) + } + + fn get_key_type(&self, key: &str) -> Result, DBError> { + // First check if key exists (handles expiration) + if self.get_storage_value(key)?.is_some() { + match self.types.get(key).map_err(|e| DBError(e.to_string()))? { + Some(data) => Ok(Some(String::from_utf8_lossy(&data).to_string())), + None => Ok(None) + } + } else { + Ok(None) + } + } + + // Hash operations + fn hset(&self, key: &str, pairs: Vec<(String, String)>) -> Result { + let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue { + value: ValueType::Hash(HashMap::new()), + expires_at: None, + }); + + let hash = match &mut storage_val.value { + ValueType::Hash(h) => h, + _ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())), + }; + + let mut new_fields = 0i64; + for (field, value) in pairs { + if !hash.contains_key(&field) { + new_fields += 1; + } + hash.insert(field, value); + } + + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(new_fields) + } + + fn hget(&self, key: &str, field: &str) -> Result, DBError> { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::Hash(h) => Ok(h.get(field).cloned()), + _ => Ok(None) + } + None => Ok(None) + } + } + + fn hgetall(&self, key: &str) -> Result, DBError> { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::Hash(h) => Ok(h.into_iter().collect()), + _ => Ok(Vec::new()) + } + None => Ok(Vec::new()) + } + } + + fn hscan(&self, key: &str, cursor: u64, pattern: Option<&str>, count: Option) -> Result<(u64, Vec<(String, String)>), DBError> { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::Hash(h) => { + let mut result = Vec::new(); + let mut current_cursor = 0u64; + let limit = count.unwrap_or(10) as usize; + + for (field, value) in h.iter() { + if current_cursor >= cursor { + let matches = if let Some(pat) = pattern { + Self::glob_match(pat, field) + } else { + true + }; + + if matches { + result.push((field.clone(), value.clone())); + if result.len() >= limit { + current_cursor += 1; + break; + } + } + } + current_cursor += 1; + } + + let next_cursor = if result.len() < limit { 0 } else { current_cursor }; + Ok((next_cursor, result)) + } + _ => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())) + } + None => Ok((0, Vec::new())) + } + } + + fn hdel(&self, key: &str, fields: Vec) -> Result { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(0) + }; + + let hash = match &mut storage_val.value { + ValueType::Hash(h) => h, + _ => return Ok(0) + }; + + let mut deleted = 0i64; + for field in fields { + if hash.remove(&field).is_some() { + deleted += 1; + } + } + + if hash.is_empty() { + self.del(key.to_string())?; + } else { + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + } + + Ok(deleted) + } + + fn hexists(&self, key: &str, field: &str) -> Result { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::Hash(h) => Ok(h.contains_key(field)), + _ => Ok(false) + } + None => Ok(false) + } + } + + fn hkeys(&self, key: &str) -> Result, DBError> { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::Hash(h) => Ok(h.keys().cloned().collect()), + _ => Ok(Vec::new()) + } + None => Ok(Vec::new()) + } + } + + fn hvals(&self, key: &str) -> Result, DBError> { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::Hash(h) => Ok(h.values().cloned().collect()), + _ => Ok(Vec::new()) + } + None => Ok(Vec::new()) + } + } + + fn hlen(&self, key: &str) -> Result { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::Hash(h) => Ok(h.len() as i64), + _ => Ok(0) + } + None => Ok(0) + } + } + + fn hmget(&self, key: &str, fields: Vec) -> Result>, DBError> { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::Hash(h) => { + Ok(fields.into_iter().map(|f| h.get(&f).cloned()).collect()) + } + _ => Ok(fields.into_iter().map(|_| None).collect()) + } + None => Ok(fields.into_iter().map(|_| None).collect()) + } + } + + fn hsetnx(&self, key: &str, field: &str, value: &str) -> Result { + let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue { + value: ValueType::Hash(HashMap::new()), + expires_at: None, + }); + + let hash = match &mut storage_val.value { + ValueType::Hash(h) => h, + _ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())), + }; + + if hash.contains_key(field) { + Ok(false) + } else { + hash.insert(field.to_string(), value.to_string()); + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(true) + } + } + + // List operations + fn lpush(&self, key: &str, elements: Vec) -> Result { + let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue { + value: ValueType::List(Vec::new()), + expires_at: None, + }); + + let list = match &mut storage_val.value { + ValueType::List(l) => l, + _ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())), + }; + + for element in elements.into_iter().rev() { + list.insert(0, element); + } + + let len = list.len() as i64; + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(len) + } + + fn rpush(&self, key: &str, elements: Vec) -> Result { + let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue { + value: ValueType::List(Vec::new()), + expires_at: None, + }); + + let list = match &mut storage_val.value { + ValueType::List(l) => l, + _ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())), + }; + + list.extend(elements); + let len = list.len() as i64; + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(len) + } + + fn lpop(&self, key: &str, count: u64) -> Result, DBError> { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(Vec::new()) + }; + + let list = match &mut storage_val.value { + ValueType::List(l) => l, + _ => return Ok(Vec::new()) + }; + + let mut result = Vec::new(); + for _ in 0..count.min(list.len() as u64) { + if let Some(elem) = list.first() { + result.push(elem.clone()); + list.remove(0); + } + } + + if list.is_empty() { + self.del(key.to_string())?; + } else { + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + } + + Ok(result) + } + + fn rpop(&self, key: &str, count: u64) -> Result, DBError> { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(Vec::new()) + }; + + let list = match &mut storage_val.value { + ValueType::List(l) => l, + _ => return Ok(Vec::new()) + }; + + let mut result = Vec::new(); + for _ in 0..count.min(list.len() as u64) { + if let Some(elem) = list.pop() { + result.push(elem); + } + } + + if list.is_empty() { + self.del(key.to_string())?; + } else { + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + } + + Ok(result) + } + + fn llen(&self, key: &str) -> Result { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::List(l) => Ok(l.len() as i64), + _ => Ok(0) + } + None => Ok(0) + } + } + + fn lindex(&self, key: &str, index: i64) -> Result, DBError> { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::List(list) => { + let actual_index = if index < 0 { + list.len() as i64 + index + } else { + index + }; + + if actual_index >= 0 && (actual_index as usize) < list.len() { + Ok(Some(list[actual_index as usize].clone())) + } else { + Ok(None) + } + } + _ => Ok(None) + } + None => Ok(None) + } + } + + fn lrange(&self, key: &str, start: i64, stop: i64) -> Result, DBError> { + match self.get_storage_value(key)? { + Some(storage_val) => match storage_val.value { + ValueType::List(list) => { + if list.is_empty() { + return Ok(Vec::new()); + } + + let len = list.len() as i64; + let start_idx = if start < 0 { + std::cmp::max(0, len + start) + } else { + std::cmp::min(start, len) + }; + let stop_idx = if stop < 0 { + std::cmp::max(-1, len + stop) + } else { + std::cmp::min(stop, len - 1) + }; + + if start_idx > stop_idx || start_idx >= len { + return Ok(Vec::new()); + } + + let start_usize = start_idx as usize; + let stop_usize = (stop_idx + 1) as usize; + + Ok(list[start_usize..std::cmp::min(stop_usize, list.len())].to_vec()) + } + _ => Ok(Vec::new()) + } + None => Ok(Vec::new()) + } + } + + fn ltrim(&self, key: &str, start: i64, stop: i64) -> Result<(), DBError> { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(()) + }; + + let list = match &mut storage_val.value { + ValueType::List(l) => l, + _ => return Ok(()) + }; + + if list.is_empty() { + return Ok(()); + } + + let len = list.len() as i64; + let start_idx = if start < 0 { + std::cmp::max(0, len + start) + } else { + std::cmp::min(start, len) + }; + let stop_idx = if stop < 0 { + std::cmp::max(-1, len + stop) + } else { + std::cmp::min(stop, len - 1) + }; + + if start_idx > stop_idx || start_idx >= len { + self.del(key.to_string())?; + } else { + let start_usize = start_idx as usize; + let stop_usize = (stop_idx + 1) as usize; + *list = list[start_usize..std::cmp::min(stop_usize, list.len())].to_vec(); + + if list.is_empty() { + self.del(key.to_string())?; + } else { + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + } + } + + Ok(()) + } + + fn lrem(&self, key: &str, count: i64, element: &str) -> Result { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(0) + }; + + let list = match &mut storage_val.value { + ValueType::List(l) => l, + _ => return Ok(0) + }; + + let mut removed = 0i64; + + if count == 0 { + // Remove all occurrences + let original_len = list.len(); + list.retain(|x| x != element); + removed = (original_len - list.len()) as i64; + } else if count > 0 { + // Remove first count occurrences + let mut to_remove = count as usize; + list.retain(|x| { + if x == element && to_remove > 0 { + to_remove -= 1; + removed += 1; + false + } else { + true + } + }); + } else { + // Remove last |count| occurrences + let mut to_remove = (-count) as usize; + for i in (0..list.len()).rev() { + if list[i] == element && to_remove > 0 { + list.remove(i); + to_remove -= 1; + removed += 1; + } + } + } + + if list.is_empty() { + self.del(key.to_string())?; + } else { + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + } + + Ok(removed) + } + + // Expiration + fn ttl(&self, key: &str) -> Result { + match self.get_storage_value(key)? { + Some(storage_val) => { + if let Some(expires_at) = storage_val.expires_at { + let now = Self::now_millis(); + if now >= expires_at { + Ok(-2) // Key has expired + } else { + Ok(((expires_at - now) / 1000) as i64) // TTL in seconds + } + } else { + Ok(-1) // Key exists but has no expiration + } + } + None => Ok(-2) // Key does not exist + } + } + + fn expire_seconds(&self, key: &str, secs: u64) -> Result { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(false) + }; + + storage_val.expires_at = Some(Self::now_millis() + (secs as u128) * 1000); + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(true) + } + + fn pexpire_millis(&self, key: &str, ms: u128) -> Result { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(false) + }; + + storage_val.expires_at = Some(Self::now_millis() + ms); + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(true) + } + + fn persist(&self, key: &str) -> Result { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(false) + }; + + if storage_val.expires_at.is_some() { + storage_val.expires_at = None; + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(true) + } else { + Ok(false) + } + } + + fn expire_at_seconds(&self, key: &str, ts_secs: i64) -> Result { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(false) + }; + + let expires_at_ms: u128 = if ts_secs <= 0 { 0 } else { (ts_secs as u128) * 1000 }; + storage_val.expires_at = Some(expires_at_ms); + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(true) + } + + fn pexpire_at_millis(&self, key: &str, ts_ms: i64) -> Result { + let mut storage_val = match self.get_storage_value(key)? { + Some(sv) => sv, + None => return Ok(false) + }; + + let expires_at_ms: u128 = if ts_ms <= 0 { 0 } else { ts_ms as u128 }; + storage_val.expires_at = Some(expires_at_ms); + self.set_storage_value(key, storage_val)?; + self.db.flush().map_err(|e| DBError(e.to_string()))?; + Ok(true) + } + + fn is_encrypted(&self) -> bool { + self.crypto.is_some() + } + + fn clone_arc(&self) -> Arc { + // Note: This is a simplified clone - in production you might want to + // handle this differently as sled::Db is already Arc internally + Arc::new(SledStorage { + db: self.db.clone(), + types: self.types.clone(), + crypto: self.crypto.clone(), + }) + } +} \ No newline at end of file diff --git a/herodb/src/storage_trait.rs b/herodb/src/storage_trait.rs new file mode 100644 index 0000000..5a8f482 --- /dev/null +++ b/herodb/src/storage_trait.rs @@ -0,0 +1,57 @@ +// src/storage_trait.rs +use crate::error::DBError; +use std::sync::Arc; + +pub trait StorageBackend: Send + Sync { + // Basic key operations + fn get(&self, key: &str) -> Result, DBError>; + fn set(&self, key: String, value: String) -> Result<(), DBError>; + fn setx(&self, key: String, value: String, expire_ms: u128) -> Result<(), DBError>; + fn del(&self, key: String) -> Result<(), DBError>; + fn exists(&self, key: &str) -> Result; + fn keys(&self, pattern: &str) -> Result, DBError>; + fn dbsize(&self) -> Result; + fn flushdb(&self) -> Result<(), DBError>; + fn get_key_type(&self, key: &str) -> Result, DBError>; + + // Scanning + fn scan(&self, cursor: u64, pattern: Option<&str>, count: Option) -> Result<(u64, Vec<(String, String)>), DBError>; + fn hscan(&self, key: &str, cursor: u64, pattern: Option<&str>, count: Option) -> Result<(u64, Vec<(String, String)>), DBError>; + + // Hash operations + fn hset(&self, key: &str, pairs: Vec<(String, String)>) -> Result; + fn hget(&self, key: &str, field: &str) -> Result, DBError>; + fn hgetall(&self, key: &str) -> Result, DBError>; + fn hdel(&self, key: &str, fields: Vec) -> Result; + fn hexists(&self, key: &str, field: &str) -> Result; + fn hkeys(&self, key: &str) -> Result, DBError>; + fn hvals(&self, key: &str) -> Result, DBError>; + fn hlen(&self, key: &str) -> Result; + fn hmget(&self, key: &str, fields: Vec) -> Result>, DBError>; + fn hsetnx(&self, key: &str, field: &str, value: &str) -> Result; + + // List operations + fn lpush(&self, key: &str, elements: Vec) -> Result; + fn rpush(&self, key: &str, elements: Vec) -> Result; + fn lpop(&self, key: &str, count: u64) -> Result, DBError>; + fn rpop(&self, key: &str, count: u64) -> Result, DBError>; + fn llen(&self, key: &str) -> Result; + fn lindex(&self, key: &str, index: i64) -> Result, DBError>; + fn lrange(&self, key: &str, start: i64, stop: i64) -> Result, DBError>; + fn ltrim(&self, key: &str, start: i64, stop: i64) -> Result<(), DBError>; + fn lrem(&self, key: &str, count: i64, element: &str) -> Result; + + // Expiration + fn ttl(&self, key: &str) -> Result; + fn expire_seconds(&self, key: &str, secs: u64) -> Result; + fn pexpire_millis(&self, key: &str, ms: u128) -> Result; + fn persist(&self, key: &str) -> Result; + fn expire_at_seconds(&self, key: &str, ts_secs: i64) -> Result; + fn pexpire_at_millis(&self, key: &str, ts_ms: i64) -> Result; + + // Metadata + fn is_encrypted(&self) -> bool; + + // Clone to Arc for sharing + fn clone_arc(&self) -> Arc; +} \ No newline at end of file