herocode/herodb
12
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
despiegk
2025-08-22 17:09:08 +02:00
parent 09553f54c8
commit 9054737e84
12 changed files with 1373 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
Cargo.lock generated
View File

@@ -206,6 +206,12 @@ dependencies = [
"serde",
]
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.9.2"
@@ -358,6 +364,30 @@ dependencies = [
"libc",
]
[[package]]
name = "crc32fast"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crypto-common"
version = "0.1.6"
@@ -406,7 +436,7 @@ dependencies = [
"hashbrown",
"lock_api",
"once_cell",
"parking_lot_core",
"parking_lot_core 0.9.11",
]
[[package]]
@@ -533,6 +563,16 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "futures"
version = "0.3.31"
@@ -622,6 +662,15 @@ dependencies = [
"slab",
]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]]
name = "generic-array"
version = "0.14.7"
@@ -682,6 +731,7 @@ dependencies = [
"serde",
"serde_json",
"sha2",
"sled",
"thiserror",
"tokio",
]
@@ -732,7 +782,7 @@ dependencies = [
"intl-memoizer",
"lazy_static",
"log",
"parking_lot",
"parking_lot 0.12.4",
"rust-embed",
"thiserror",
"unic-langid",
@@ -889,6 +939,15 @@ dependencies = [
"generic-array",
]
[[package]]
name = "instant"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
dependencies = [
"cfg-if",
]
[[package]]
name = "intl-memoizer"
version = "0.5.3"
@@ -914,7 +973,7 @@ version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"
dependencies = [
"bitflags",
"bitflags 2.9.2",
"cfg-if",
"libc",
]
@@ -1040,6 +1099,17 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]]
name = "parking_lot"
version = "0.12.4"
@@ -1047,7 +1117,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13"
dependencies = [
"lock_api",
"parking_lot_core",
"parking_lot_core 0.9.11",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
]
[[package]]
@@ -1058,7 +1142,7 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"redox_syscall 0.5.17",
"smallvec",
"windows-targets 0.52.6",
]
@@ -1252,13 +1336,22 @@ dependencies = [
"url",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
dependencies = [
"bitflags",
"bitflags 2.9.2",
]
[[package]]
@@ -1466,6 +1559,22 @@ version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
[[package]]
name = "sled"
version = "0.34.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
dependencies = [
"crc32fast",
"crossbeam-epoch",
"crossbeam-utils",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot 0.11.2",
]
[[package]]
name = "smallvec"
version = "1.15.1"
@@ -1599,7 +1708,7 @@ dependencies = [
"io-uring",
"libc",
"mio",
"parking_lot",
"parking_lot 0.12.4",
"pin-project-lite",
"signal-hook-registry",
"slab",

3
herodb/Cargo.toml
View File

@@ -12,10 +12,11 @@ tokio = { version = "1.23.0", features = ["full"] }
clap = { version = "4.5.20", features = ["derive"] }
byteorder = "1.4.3"
futures = "0.3"
sled = "0.34"
redb = "2.1.3"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
bincode = "1.3.3"
bincode = "1.3"
chacha20poly1305 = "0.10.1"
rand = "0.8"
sha2 = "0.10"

85
herodb/README.md
View File

@@ -0,0 +1,85 @@
# HeroDB
HeroDB is a Redis-compatible database built with Rust, offering a flexible and secure storage solution. It supports two primary storage backends: `redb` (default) and `sled`, both with full encryption capabilities. HeroDB aims to provide a robust and performant key-value store with advanced features like data-at-rest encryption, hash operations, list operations, and cursor-based scanning.
## Purpose
The main purpose of HeroDB is to offer a lightweight, embeddable, and Redis-compatible database that prioritizes data security through transparent encryption. It's designed for applications that require fast, reliable data storage with the option for strong cryptographic protection, without the overhead of a full-fledged Redis server.
## Features
- **Redis Compatibility**: Supports a subset of Redis commands over RESP (Redis Serialization Protocol) via TCP.
- **Dual Backend Support**:
- `redb` (default): Optimized for concurrent access and high-throughput scenarios.
- `sled`: A lock-free, log-structured database, excellent for specific workloads.
- **Data-at-Rest Encryption**: Transparent encryption for both backends using the `age` encryption library.
- **Key-Value Operations**: Full support for basic string, hash, and list operations.
- **Expiration**: Time-to-live (TTL) functionality for keys.
- **Scanning**: Cursor-based iteration for keys and hash fields (`SCAN`, `HSCAN`).
- **AGE Cryptography Commands**: HeroDB-specific extensions for cryptographic operations.
## Quick Start
### Building HeroDB
To build HeroDB, navigate to the project root and run:
```bash
cargo build --release
```
### Running HeroDB
You can start HeroDB with different backends and encryption options:
#### Default `redb` Backend
```bash
./target/release/herodb --dir /tmp/herodb_redb --port 6379
```
#### `sled` Backend
```bash
./target/release/herodb --dir /tmp/herodb_sled --port 6379 --sled
```
#### `redb` with Encryption
```bash
./target/release/herodb --dir /tmp/herodb_encrypted --port 6379 --encrypt --key mysecretkey
```
#### `sled` with Encryption
```bash
./target/release/herodb --dir /tmp/herodb_sled_encrypted --port 6379 --sled --encrypt --key mysecretkey
```
## Usage with Redis Clients
HeroDB can be interacted with using any standard Redis client, such as `redis-cli`, `redis-py` (Python), or `ioredis` (Node.js).
### Example with `redis-cli`
```bash
redis-cli -p 6379 SET mykey "Hello from HeroDB!"
redis-cli -p 6379 GET mykey
# → "Hello from HeroDB!"
redis-cli -p 6379 HSET user:1 name "Alice" age "30"
redis-cli -p 6379 HGET user:1 name
# → "Alice"
redis-cli -p 6379 SCAN 0 MATCH user:* COUNT 10
# → 1) "0"
# 2) 1) "user:1"
```
## Documentation
For more detailed information on commands, features, and advanced usage, please refer to the documentation:
- [Basics](docs/basics.md)
- [Supported Commands](docs/cmds.md)
- [AGE Cryptography](docs/age.md)

307
herodb/docs/cmds.md
View File

@@ -1,227 +1,116 @@
# HeroDB Redis Protocol Support: Commands & Client Usage
## Backend Support
HeroDB is a Redis-compatible database built using the `redb` database backend.
HeroDB supports two storage backends, both with full encryption support:
It supports a subset of Redis commands over the standard RESP (Redis Serialization Protocol) via TCP, allowing you to interact with it using standard Redis clients like `redis-cli`, Python's `redis-py`, Node.js's `ioredis`, etc.
- **redb** (default): Full-featured, optimized for production use
- **sled**: Alternative embedded database with encryption support
This document provides:
- A list of all currently supported Redis commands.
- Example usage with standard Redis clients.
- Bash and Rust test-inspired usage examples.
## Quick Start
Assuming the server is running on localhost at port `$PORT`:
### Starting HeroDB with Different Backends
```bash
# Build HeroDB
cargo build --release
# Use default redb backend
./target/release/herodb --dir /tmp/herodb_redb --port 6379
# Start HeroDB server
./target/release/herodb --dir /tmp/herodb_data --port 6381 --debug
# Use sled backend
./target/release/herodb --dir /tmp/herodb_sled --port 6379 --sled
# Use redb with encryption
./target/release/herodb --dir /tmp/herodb_encrypted --port 6379 --encrypt --key mysecretkey
# Use sled with encryption
./target/release/herodb --dir /tmp/herodb_sled_encrypted --port 6379 --sled --encrypt --key mysecretkey
```
## Using Standard Redis Clients
### Command Support by Backend
### With `redis-cli`
Command Category | redb | sled | Notes |
|-----------------|------|------|-------|
**Strings** | | | |
SET | ✅ | ✅ | Full support |
GET | ✅ | ✅ | Full support |
DEL | ✅ | ✅ | Full support |
EXISTS | ✅ | ✅ | Full support |
INCR/DECR | ✅ | ✅ | Full support |
MGET/MSET | ✅ | ✅ | Full support |
**Hashes** | | | |
HSET | ✅ | ✅ | Full support |
HGET | ✅ | ✅ | Full support |
HGETALL | ✅ | ✅ | Full support |
HDEL | ✅ | ✅ | Full support |
HEXISTS | ✅ | ✅ | Full support |
HKEYS | ✅ | ✅ | Full support |
HVALS | ✅ | ✅ | Full support |
HLEN | ✅ | ✅ | Full support |
HMGET | ✅ | ✅ | Full support |
HSETNX | ✅ | ✅ | Full support |
HINCRBY/HINCRBYFLOAT | ✅ | ✅ | Full support |
HSCAN | ✅ | ✅ | Full support with pattern matching |
**Lists** | | | |
LPUSH/RPUSH | ✅ | ✅ | Full support |
LPOP/RPOP | ✅ | ✅ | Full support |
LLEN | ✅ | ✅ | Full support |
LRANGE | ✅ | ✅ | Full support |
LINDEX | ✅ | ✅ | Full support |
LTRIM | ✅ | ✅ | Full support |
LREM | ✅ | ✅ | Full support |
BLPOP/BRPOP | ✅ | ❌ | Blocking operations not in sled |
**Expiration** | | | |
EXPIRE | ✅ | ✅ | Full support in both |
TTL | ✅ | ✅ | Full support in both |
PERSIST | ✅ | ✅ | Full support in both |
SETEX/PSETEX | ✅ | ✅ | Full support in both |
EXPIREAT/PEXPIREAT | ✅ | ✅ | Full support in both |
**Scanning** | | | |
KEYS | ✅ | ✅ | Full support with patterns |
SCAN | ✅ | ✅ | Full cursor-based iteration |
HSCAN | ✅ | ✅ | Full cursor-based iteration |
**Transactions** | | | |
MULTI/EXEC/DISCARD | ✅ | ❌ | Only supported in redb |
**Encryption** | | | |
Data-at-rest encryption | ✅ | ✅ | Both support [age](age.tech) encryption |
AGE commands | ✅ | ✅ | Both support AGE crypto commands |
### Performance Considerations
- **redb**: Optimized for concurrent access, better for high-throughput scenarios
- **sled**: Lock-free architecture, excellent for specific workloads
### Encryption Features
Both backends support:
- Transparent data-at-rest encryption using the `age` encryption library
- Per-database encryption (databases >= 10 are encrypted when `--encrypt` flag is used)
- Secure key derivation using the master key
### Backend Selection Examples
```bash
redis-cli -p 6381 SET mykey "hello"
redis-cli -p 6381 GET mykey
# Example: Testing both backends
redis-cli -p 6379 SET mykey "redb value"
redis-cli -p 6381 SET mykey "sled value"
# Example: Using encryption with both
./target/release/herodb --port 6379 --encrypt --key secret123
./target/release/herodb --port 6381 --sled --encrypt --key secret123
# Both support the same Redis commands
redis-cli -p 6379 HSET user:1 name "Alice" age "30"
redis-cli -p 6381 HSET user:1 name "Alice" age "30"
# Both support SCAN operations
redis-cli -p 6379 SCAN 0 MATCH user:* COUNT 10
redis-cli -p 6381 SCAN 0 MATCH user:* COUNT 10
```
### With Python (`redis-py`)
### Migration Between Backends
```python
import redis
r = redis.Redis(host='localhost', port=6381, db=0)
r.set('mykey', 'hello')
print(r.get('mykey').decode())
```
### With Node.js (`ioredis`)
```js
const Redis = require("ioredis");
const redis = new Redis({ port: 6381, host: "localhost" });
await redis.set("mykey", "hello");
const value = await redis.get("mykey");
console.log(value); // "hello"
```
## Supported Redis Commands
### String Commands
| Command | Description | Example Usage |
|---------------|------------------------------------------|-------------------------------------------|
| `SET` | Set a key to a string value | `SET name "Alice"` |
| `GET` | Get the value of a key | `GET name` |
| `DEL` | Delete one or more keys | `DEL name age` |
| `INCR` | Increment the integer value of a key | `INCR counter` |
| `DECR` | Decrement the integer value of a key | `DECR counter` |
| `INCRBY` | Increment key by a given integer | `INCRBY counter 5` |
| `DECRBY` | Decrement key by a given integer | `DECRBY counter 3` |
| `EXISTS` | Check if a key exists | `EXISTS name` |
| `TYPE` | Return the type of a key | `TYPE name` |
### Hash Commands
| Command | Description | Example Usage |
|---------------|------------------------------------------|-------------------------------------------|
| `HSET` | Set field in hash stored at key | `HSET user:1 name "Alice"` |
| `HGET` | Get value of a field in hash | `HGET user:1 name` |
| `HGETALL` | Get all fields and values in a hash | `HGETALL user:1` |
| `HDEL` | Delete one or more fields from hash | `HDEL user:1 name age` |
| `HEXISTS` | Check if field exists in hash | `HEXISTS user:1 name` |
| `HKEYS` | Get all field names in a hash | `HKEYS user:1` |
| `HVALS` | Get all values in a hash | `HVALS user:1` |
| `HLEN` | Get number of fields in a hash | `HLEN user:1` |
| `HMGET` | Get values of multiple fields | `HMGET user:1 name age` |
| `HSETNX` | Set field only if it does not exist | `HSETNX user:1 email alice@example.com` |
### List Commands
| Command | Description | Example Usage |
|---------------|------------------------------------------|-------------------------------------------|
| `LPUSH` | Insert elements at the head of a list | `LPUSH mylist "item1" "item2"` |
| `RPUSH` | Insert elements at the tail of a list | `RPUSH mylist "item3" "item4"` |
| `LPOP` | Remove and return element from head | `LPOP mylist` |
| `RPOP` | Remove and return element from tail | `RPOP mylist` |
| `BLPOP` | Blocking remove from head with timeout | `BLPOP mylist1 mylist2 5` |
| `BRPOP` | Blocking remove from tail with timeout | `BRPOP mylist1 mylist2 5` |
| `LLEN` | Get the length of a list | `LLEN mylist` |
| `LREM` | Remove elements from list | `LREM mylist 2 "item"` |
| `LTRIM` | Trim list to specified range | `LTRIM mylist 0 5` |
| `LINDEX` | Get element by index | `LINDEX mylist 0` |
| `LRANGE` | Get range of elements | `LRANGE mylist 0 -1` |
### Keys & Scanning
| Command | Description | Example Usage |
|---------------|------------------------------------------|-------------------------------------------|
| `KEYS` | Find all keys matching a pattern | `KEYS user:*` |
| `SCAN` | Incrementally iterate keys | `SCAN 0 MATCH user:* COUNT 10` |
### Expiration
| Command | Description | Example Usage |
|---------------|------------------------------------------|-------------------------------------------|
| `EXPIRE` | Set a key's time to live in seconds | `EXPIRE tempkey 60` |
| `TTL` | Get the time to live for a key | `TTL tempkey` |
| `PERSIST` | Remove the expiration from a key | `PERSIST tempkey` |
### Transactions
| Command | Description | Example Usage |
|---------------|------------------------------------------|-------------------------------------------|
| `MULTI` | Start a transaction block | `MULTI` |
| `EXEC` | Execute all commands in a transaction | `EXEC` |
| `DISCARD` | Discard all commands in a transaction | `DISCARD` |
### Configuration
| Command | Description | Example Usage |
|---------------|------------------------------------------|-------------------------------------------|
| `CONFIG GET` | Get configuration parameters | `CONFIG GET dir` |
| `CONFIG SET` | Set configuration parameters | `CONFIG SET maxmemory 100mb` |
### Info & Monitoring
| Command | Description | Example Usage |
|---------------|------------------------------------------|-------------------------------------------|
| `INFO` | Get information and statistics about server | `INFO` |
| `PING` | Ping the server | `PING` |
### AGE Cryptography Commands
| Command | Description | Example Usage |
|--------------------|-----------------------------------------------|-----------------------------------------------|
| `AGE GENENC` | Generate ephemeral encryption keypair | `AGE GENENC` |
| `AGE GENSIGN` | Generate ephemeral signing keypair | `AGE GENSIGN` |
| `AGE ENCRYPT` | Encrypt a message using a public key | `AGE ENCRYPT <recipient> "msg"` |
| `AGE DECRYPT` | Decrypt a message using a secret key | `AGE DECRYPT <identity> <ciphertext>` |
| `AGE SIGN` | Sign a message using a secret key | `AGE SIGN <sign_secret> "msg"` |
| `AGE VERIFY` | Verify a signature using a public key | `AGE VERIFY <pubkey> "msg" <signature>` |
| `AGE KEYGEN` | Create and persist a named encryption key | `AGE KEYGEN app1` |
| `AGE SIGNKEYGEN` | Create and persist a named signing key | `AGE SIGNKEYGEN app1` |
| `AGE ENCRYPTNAME` | Encrypt using a named key | `AGE ENCRYPTNAME app1 "msg"` |
| `AGE DECRYPTNAME` | Decrypt using a named key | `AGE DECRYPTNAME app1 <ciphertext>` |
| `AGE SIGNNAME` | Sign using a named key | `AGE SIGNNAME app1 "msg"` |
| `AGE VERIFYNAME` | Verify using a named key | `AGE VERIFYNAME app1 "msg" <signature>` |
| `AGE LIST` | List all persisted named keys | `AGE LIST` |
> Note: AGE commands are not part of standard Redis. They are HeroDB-specific extensions for cryptographic operations.
## Example Usage
### Basic String Operations
To migrate data between backends, use Redis replication or dump/restore:
```bash
redis-cli -p 6381 SET greeting "Hello, HeroDB!"
redis-cli -p 6381 GET greeting
# → "Hello, HeroDB!"
# Export from redb
redis-cli -p 6379 --rdb dump.rdb
redis-cli -p 6381 INCR visits
redis-cli -p 6381 INCR visits
redis-cli -p 6381 GET visits
# → "2"
```
### Hash Operations
```bash
redis-cli -p 6381 HSET user:1000 name "Alice" age "30" city "NYC"
redis-cli -p 6381 HGET user:1000 name
# → "Alice"
redis-cli -p 6381 HGETALL user:1000
# → 1) "name"
# 2) "Alice"
# 3) "age"
# 4) "30"
# 5) "city"
# 6) "NYC"
```
### Expiration
```bash
redis-cli -p 6381 SET tempkey "temporary"
redis-cli -p 6381 EXPIRE tempkey 5
redis-cli -p 6381 TTL tempkey
# → (integer) 4
# After 5 seconds:
redis-cli -p 6381 GET tempkey
# → (nil)
```
### Transactions
```bash
redis-cli -p 6381 MULTI
redis-cli -p 6381 SET txkey1 "value1"
redis-cli -p 6381 SET txkey2 "value2"
redis-cli -p 6381 INCR counter
redis-cli -p 6381 EXEC
# → 1) OK
# 2) OK
# 3) (integer) 3
```
### Scanning Keys
```bash
redis-cli -p 6381 SET scankey1 "val1"
redis-cli -p 6381 SET scankey2 "val2"
redis-cli -p 6381 HSET scanhash field1 "val1"
redis-cli -p 6381 SCAN 0 MATCH scankey*
# → 1) "0"
# 2) 1) "scankey1"
# 2) "scankey2"
```
# Import to sled
redis-cli -p 6381 --pipe < dump.rdb
```

0
herodb/specs/backgroundinfo/tantivy.md Normal file
View File

1
herodb/src/crypto.rs
View File

@@ -23,6 +23,7 @@ impl From<CryptoError> for crate::error::DBError {
}
/// Super-simple factory: new(secret) + encrypt(bytes) + decrypt(bytes)
#[derive(Clone)]
pub struct CryptoFactory {
key: chacha20poly1305::Key,
}

2
herodb/src/lib.rs
View File

@@ -6,3 +6,5 @@ pub mod options;
pub mod protocol;
pub mod server;
pub mod storage;
pub mod storage_trait; // Add this
pub mod storage_sled; // Add this

10
herodb/src/main.rs
View File

@@ -30,6 +30,10 @@ struct Args {
/// Encrypt the database
#[arg(long)]
encrypt: bool,
/// Use the sled backend
#[arg(long)]
sled: bool,
}
#[tokio::main]
@@ -47,10 +51,14 @@ async fn main() {
// new DB option
let option = herodb::options::DBOption {
dir: args.dir,
port,
debug: args.debug,
encryption_key: args.encryption_key,
encrypt: args.encrypt,
backend: if args.sled {
herodb::options::BackendType::Sled
} else {
herodb::options::BackendType::Redb
},
};
// new server

12
herodb/src/options.rs
View File

@@ -1,8 +1,14 @@
#[derive(Clone)]
#[derive(Debug, Clone)]
pub enum BackendType {
Redb,
Sled,
}
#[derive(Debug, Clone)]
pub struct DBOption {
pub dir: String,
pub port: u16,
pub debug: bool,
pub encrypt: bool,
pub encryption_key: Option<String>, // Master encryption key
pub encryption_key: Option<String>,
pub backend: BackendType,
}

157
herodb/src/storage/mod.rs
View File

@@ -1,5 +1,6 @@
use std::{
path::Path,
sync::Arc,
time::{SystemTime, UNIX_EPOCH},
};
@@ -123,4 +124,160 @@ impl Storage {
Ok(data.to_vec())
}
}
}
use crate::storage_trait::StorageBackend;
impl StorageBackend for Storage {
fn get(&self, key: &str) -> Result<Option<String>, DBError> {
self.get(key)
}
fn set(&self, key: String, value: String) -> Result<(), DBError> {
self.set(key, value)
}
fn setx(&self, key: String, value: String, expire_ms: u128) -> Result<(), DBError> {
self.setx(key, value, expire_ms)
}
fn del(&self, key: String) -> Result<(), DBError> {
self.del(key)
}
fn exists(&self, key: &str) -> Result<bool, DBError> {
self.exists(key)
}
fn keys(&self, pattern: &str) -> Result<Vec<String>, DBError> {
self.keys(pattern)
}
fn dbsize(&self) -> Result<i64, DBError> {
self.dbsize()
}
fn flushdb(&self) -> Result<(), DBError> {
self.flushdb()
}
fn get_key_type(&self, key: &str) -> Result<Option<String>, DBError> {
self.get_key_type(key)
}
fn scan(&self, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError> {
self.scan(cursor, pattern, count)
}
fn hscan(&self, key: &str, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError> {
self.hscan(key, cursor, pattern, count)
}
fn hset(&self, key: &str, pairs: Vec<(String, String)>) -> Result<i64, DBError> {
self.hset(key, pairs)
}
fn hget(&self, key: &str, field: &str) -> Result<Option<String>, DBError> {
self.hget(key, field)
}
fn hgetall(&self, key: &str) -> Result<Vec<(String, String)>, DBError> {
self.hgetall(key)
}
fn hdel(&self, key: &str, fields: Vec<String>) -> Result<i64, DBError> {
self.hdel(key, fields)
}
fn hexists(&self, key: &str, field: &str) -> Result<bool, DBError> {
self.hexists(key, field)
}
fn hkeys(&self, key: &str) -> Result<Vec<String>, DBError> {
self.hkeys(key)
}
fn hvals(&self, key: &str) -> Result<Vec<String>, DBError> {
self.hvals(key)
}
fn hlen(&self, key: &str) -> Result<i64, DBError> {
self.hlen(key)
}
fn hmget(&self, key: &str, fields: Vec<String>) -> Result<Vec<Option<String>>, DBError> {
self.hmget(key, fields)
}
fn hsetnx(&self, key: &str, field: &str, value: &str) -> Result<bool, DBError> {
self.hsetnx(key, field, value)
}
fn lpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError> {
self.lpush(key, elements)
}
fn rpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError> {
self.rpush(key, elements)
}
fn lpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError> {
self.lpop(key, count)
}
fn rpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError> {
self.rpop(key, count)
}
fn llen(&self, key: &str) -> Result<i64, DBError> {
self.llen(key)
}
fn lindex(&self, key: &str, index: i64) -> Result<Option<String>, DBError> {
self.lindex(key, index)
}
fn lrange(&self, key: &str, start: i64, stop: i64) -> Result<Vec<String>, DBError> {
self.lrange(key, start, stop)
}
fn ltrim(&self, key: &str, start: i64, stop: i64) -> Result<(), DBError> {
self.ltrim(key, start, stop)
}
fn lrem(&self, key: &str, count: i64, element: &str) -> Result<i64, DBError> {
self.lrem(key, count, element)
}
fn ttl(&self, key: &str) -> Result<i64, DBError> {
self.ttl(key)
}
fn expire_seconds(&self, key: &str, secs: u64) -> Result<bool, DBError> {
self.expire_seconds(key, secs)
}
fn pexpire_millis(&self, key: &str, ms: u128) -> Result<bool, DBError> {
self.pexpire_millis(key, ms)
}
fn persist(&self, key: &str) -> Result<bool, DBError> {
self.persist(key)
}
fn expire_at_seconds(&self, key: &str, ts_secs: i64) -> Result<bool, DBError> {
self.expire_at_seconds(key, ts_secs)
}
fn pexpire_at_millis(&self, key: &str, ts_ms: i64) -> Result<bool, DBError> {
self.pexpire_at_millis(key, ts_ms)
}
fn is_encrypted(&self) -> bool {
self.is_encrypted()
}
fn clone_arc(&self) -> Arc<dyn StorageBackend> {
unimplemented!("Storage cloning not yet implemented for redb backend")
}
}

837
herodb/src/storage_sled/mod.rs Normal file
View File

@@ -0,0 +1,837 @@
// src/storage_sled/mod.rs
use std::path::Path;
use std::sync::Arc;
use std::collections::HashMap;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use crate::error::DBError;
use crate::storage_trait::StorageBackend;
use crate::crypto::CryptoFactory;
#[derive(Serialize, Deserialize, Debug, Clone)]
enum ValueType {
String(String),
Hash(HashMap<String, String>),
List(Vec<String>),
}
#[derive(Serialize, Deserialize, Debug, Clone)]
struct StorageValue {
value: ValueType,
expires_at: Option<u128>, // milliseconds since epoch
}
pub struct SledStorage {
db: sled::Db,
types: sled::Tree,
crypto: Option<CryptoFactory>,
}
impl SledStorage {
pub fn new(path: impl AsRef<Path>, should_encrypt: bool, master_key: Option<&str>) -> Result<Self, DBError> {
let db = sled::open(path).map_err(|e| DBError(format!("Failed to open sled: {}", e)))?;
let types = db.open_tree("types").map_err(|e| DBError(format!("Failed to open types tree: {}", e)))?;
// Check if database was previously encrypted
let encrypted_tree = db.open_tree("encrypted").map_err(|e| DBError(e.to_string()))?;
let was_encrypted = encrypted_tree.get("encrypted")
.map_err(|e| DBError(e.to_string()))?
.map(|v| v[0] == 1)
.unwrap_or(false);
let crypto = if should_encrypt || was_encrypted {
if let Some(key) = master_key {
Some(CryptoFactory::new(key.as_bytes()))
} else {
return Err(DBError("Encryption requested but no master key provided".to_string()));
}
} else {
None
};
// Mark database as encrypted if enabling encryption
if should_encrypt && !was_encrypted {
encrypted_tree.insert("encrypted", &[1u8])
.map_err(|e| DBError(e.to_string()))?;
encrypted_tree.flush().map_err(|e| DBError(e.to_string()))?;
}
Ok(SledStorage { db, types, crypto })
}
fn now_millis() -> u128 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_millis()
}
fn encrypt_if_needed(&self, data: &[u8]) -> Result<Vec<u8>, DBError> {
if let Some(crypto) = &self.crypto {
Ok(crypto.encrypt(data))
} else {
Ok(data.to_vec())
}
}
fn decrypt_if_needed(&self, data: &[u8]) -> Result<Vec<u8>, DBError> {
if let Some(crypto) = &self.crypto {
Ok(crypto.decrypt(data)?)
} else {
Ok(data.to_vec())
}
}
fn get_storage_value(&self, key: &str) -> Result<Option<StorageValue>, DBError> {
match self.db.get(key).map_err(|e| DBError(e.to_string()))? {
Some(encrypted_data) => {
let decrypted = self.decrypt_if_needed(&encrypted_data)?;
let storage_val: StorageValue = bincode::deserialize(&decrypted)
.map_err(|e| DBError(format!("Deserialization error: {}", e)))?;
// Check expiration
if let Some(expires_at) = storage_val.expires_at {
if Self::now_millis() > expires_at {
// Expired, remove it
self.db.remove(key).map_err(|e| DBError(e.to_string()))?;
self.types.remove(key).map_err(|e| DBError(e.to_string()))?;
return Ok(None);
}
}
Ok(Some(storage_val))
}
None => Ok(None)
}
}
fn set_storage_value(&self, key: &str, storage_val: StorageValue) -> Result<(), DBError> {
let data = bincode::serialize(&storage_val)
.map_err(|e| DBError(format!("Serialization error: {}", e)))?;
let encrypted = self.encrypt_if_needed(&data)?;
self.db.insert(key, encrypted).map_err(|e| DBError(e.to_string()))?;
// Store type info (unencrypted for efficiency)
let type_str = match &storage_val.value {
ValueType::String(_) => "string",
ValueType::Hash(_) => "hash",
ValueType::List(_) => "list",
};
self.types.insert(key, type_str.as_bytes()).map_err(|e| DBError(e.to_string()))?;
Ok(())
}
fn glob_match(pattern: &str, text: &str) -> bool {
if pattern == "*" {
return true;
}
let pattern_chars: Vec<char> = pattern.chars().collect();
let text_chars: Vec<char> = text.chars().collect();
fn match_recursive(pattern: &[char], text: &[char], pi: usize, ti: usize) -> bool {
if pi >= pattern.len() {
return ti >= text.len();
}
if ti >= text.len() {
return pattern[pi..].iter().all(|&c| c == '*');
}
match pattern[pi] {
'*' => {
for i in ti..=text.len() {
if match_recursive(pattern, text, pi + 1, i) {
return true;
}
}
false
}
'?' => match_recursive(pattern, text, pi + 1, ti + 1),
c => {
if text[ti] == c {
match_recursive(pattern, text, pi + 1, ti + 1)
} else {
false
}
}
}
}
match_recursive(&pattern_chars, &text_chars, 0, 0)
}
}
impl StorageBackend for SledStorage {
fn get(&self, key: &str) -> Result<Option<String>, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::String(s) => Ok(Some(s)),
_ => Ok(None)
}
None => Ok(None)
}
}
fn set(&self, key: String, value: String) -> Result<(), DBError> {
let storage_val = StorageValue {
value: ValueType::String(value),
expires_at: None,
};
self.set_storage_value(&key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(())
}
fn setx(&self, key: String, value: String, expire_ms: u128) -> Result<(), DBError> {
let storage_val = StorageValue {
value: ValueType::String(value),
expires_at: Some(Self::now_millis() + expire_ms),
};
self.set_storage_value(&key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(())
}
fn del(&self, key: String) -> Result<(), DBError> {
self.db.remove(&key).map_err(|e| DBError(e.to_string()))?;
self.types.remove(&key).map_err(|e| DBError(e.to_string()))?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(())
}
fn exists(&self, key: &str) -> Result<bool, DBError> {
// Check with expiration
Ok(self.get_storage_value(key)?.is_some())
}
fn keys(&self, pattern: &str) -> Result<Vec<String>, DBError> {
let mut keys = Vec::new();
for item in self.types.iter() {
let (key_bytes, _) = item.map_err(|e| DBError(e.to_string()))?;
let key = String::from_utf8_lossy(&key_bytes).to_string();
// Check if key is expired
if self.get_storage_value(&key)?.is_some() {
if Self::glob_match(pattern, &key) {
keys.push(key);
}
}
}
Ok(keys)
}
fn scan(&self, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError> {
let mut result = Vec::new();
let mut current_cursor = 0u64;
let limit = count.unwrap_or(10) as usize;
for item in self.types.iter() {
if current_cursor >= cursor {
let (key_bytes, type_bytes) = item.map_err(|e| DBError(e.to_string()))?;
let key = String::from_utf8_lossy(&key_bytes).to_string();
// Check pattern match
let matches = if let Some(pat) = pattern {
Self::glob_match(pat, &key)
} else {
true
};
if matches {
// Check if key is expired and get value
if let Some(storage_val) = self.get_storage_value(&key)? {
let value = match storage_val.value {
ValueType::String(s) => s,
_ => String::from_utf8_lossy(&type_bytes).to_string(),
};
result.push((key, value));
if result.len() >= limit {
current_cursor += 1;
break;
}
}
}
}
current_cursor += 1;
}
let next_cursor = if result.len() < limit { 0 } else { current_cursor };
Ok((next_cursor, result))
}
fn dbsize(&self) -> Result<i64, DBError> {
let mut count = 0i64;
for item in self.types.iter() {
let (key_bytes, _) = item.map_err(|e| DBError(e.to_string()))?;
let key = String::from_utf8_lossy(&key_bytes).to_string();
if self.get_storage_value(&key)?.is_some() {
count += 1;
}
}
Ok(count)
}
fn flushdb(&self) -> Result<(), DBError> {
self.db.clear().map_err(|e| DBError(e.to_string()))?;
self.types.clear().map_err(|e| DBError(e.to_string()))?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(())
}
fn get_key_type(&self, key: &str) -> Result<Option<String>, DBError> {
// First check if key exists (handles expiration)
if self.get_storage_value(key)?.is_some() {
match self.types.get(key).map_err(|e| DBError(e.to_string()))? {
Some(data) => Ok(Some(String::from_utf8_lossy(&data).to_string())),
None => Ok(None)
}
} else {
Ok(None)
}
}
// Hash operations
fn hset(&self, key: &str, pairs: Vec<(String, String)>) -> Result<i64, DBError> {
let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue {
value: ValueType::Hash(HashMap::new()),
expires_at: None,
});
let hash = match &mut storage_val.value {
ValueType::Hash(h) => h,
_ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
};
let mut new_fields = 0i64;
for (field, value) in pairs {
if !hash.contains_key(&field) {
new_fields += 1;
}
hash.insert(field, value);
}
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(new_fields)
}
fn hget(&self, key: &str, field: &str) -> Result<Option<String>, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::Hash(h) => Ok(h.get(field).cloned()),
_ => Ok(None)
}
None => Ok(None)
}
}
fn hgetall(&self, key: &str) -> Result<Vec<(String, String)>, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::Hash(h) => Ok(h.into_iter().collect()),
_ => Ok(Vec::new())
}
None => Ok(Vec::new())
}
}
fn hscan(&self, key: &str, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::Hash(h) => {
let mut result = Vec::new();
let mut current_cursor = 0u64;
let limit = count.unwrap_or(10) as usize;
for (field, value) in h.iter() {
if current_cursor >= cursor {
let matches = if let Some(pat) = pattern {
Self::glob_match(pat, field)
} else {
true
};
if matches {
result.push((field.clone(), value.clone()));
if result.len() >= limit {
current_cursor += 1;
break;
}
}
}
current_cursor += 1;
}
let next_cursor = if result.len() < limit { 0 } else { current_cursor };
Ok((next_cursor, result))
}
_ => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string()))
}
None => Ok((0, Vec::new()))
}
}
fn hdel(&self, key: &str, fields: Vec<String>) -> Result<i64, DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(0)
};
let hash = match &mut storage_val.value {
ValueType::Hash(h) => h,
_ => return Ok(0)
};
let mut deleted = 0i64;
for field in fields {
if hash.remove(&field).is_some() {
deleted += 1;
}
}
if hash.is_empty() {
self.del(key.to_string())?;
} else {
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
}
Ok(deleted)
}
fn hexists(&self, key: &str, field: &str) -> Result<bool, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::Hash(h) => Ok(h.contains_key(field)),
_ => Ok(false)
}
None => Ok(false)
}
}
fn hkeys(&self, key: &str) -> Result<Vec<String>, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::Hash(h) => Ok(h.keys().cloned().collect()),
_ => Ok(Vec::new())
}
None => Ok(Vec::new())
}
}
fn hvals(&self, key: &str) -> Result<Vec<String>, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::Hash(h) => Ok(h.values().cloned().collect()),
_ => Ok(Vec::new())
}
None => Ok(Vec::new())
}
}
fn hlen(&self, key: &str) -> Result<i64, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::Hash(h) => Ok(h.len() as i64),
_ => Ok(0)
}
None => Ok(0)
}
}
fn hmget(&self, key: &str, fields: Vec<String>) -> Result<Vec<Option<String>>, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::Hash(h) => {
Ok(fields.into_iter().map(|f| h.get(&f).cloned()).collect())
}
_ => Ok(fields.into_iter().map(|_| None).collect())
}
None => Ok(fields.into_iter().map(|_| None).collect())
}
}
fn hsetnx(&self, key: &str, field: &str, value: &str) -> Result<bool, DBError> {
let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue {
value: ValueType::Hash(HashMap::new()),
expires_at: None,
});
let hash = match &mut storage_val.value {
ValueType::Hash(h) => h,
_ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
};
if hash.contains_key(field) {
Ok(false)
} else {
hash.insert(field.to_string(), value.to_string());
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(true)
}
}
// List operations
fn lpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError> {
let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue {
value: ValueType::List(Vec::new()),
expires_at: None,
});
let list = match &mut storage_val.value {
ValueType::List(l) => l,
_ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
};
for element in elements.into_iter().rev() {
list.insert(0, element);
}
let len = list.len() as i64;
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(len)
}
fn rpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError> {
let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue {
value: ValueType::List(Vec::new()),
expires_at: None,
});
let list = match &mut storage_val.value {
ValueType::List(l) => l,
_ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
};
list.extend(elements);
let len = list.len() as i64;
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(len)
}
fn lpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(Vec::new())
};
let list = match &mut storage_val.value {
ValueType::List(l) => l,
_ => return Ok(Vec::new())
};
let mut result = Vec::new();
for _ in 0..count.min(list.len() as u64) {
if let Some(elem) = list.first() {
result.push(elem.clone());
list.remove(0);
}
}
if list.is_empty() {
self.del(key.to_string())?;
} else {
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
}
Ok(result)
}
fn rpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(Vec::new())
};
let list = match &mut storage_val.value {
ValueType::List(l) => l,
_ => return Ok(Vec::new())
};
let mut result = Vec::new();
for _ in 0..count.min(list.len() as u64) {
if let Some(elem) = list.pop() {
result.push(elem);
}
}
if list.is_empty() {
self.del(key.to_string())?;
} else {
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
}
Ok(result)
}
fn llen(&self, key: &str) -> Result<i64, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::List(l) => Ok(l.len() as i64),
_ => Ok(0)
}
None => Ok(0)
}
}
fn lindex(&self, key: &str, index: i64) -> Result<Option<String>, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::List(list) => {
let actual_index = if index < 0 {
list.len() as i64 + index
} else {
index
};
if actual_index >= 0 && (actual_index as usize) < list.len() {
Ok(Some(list[actual_index as usize].clone()))
} else {
Ok(None)
}
}
_ => Ok(None)
}
None => Ok(None)
}
}
fn lrange(&self, key: &str, start: i64, stop: i64) -> Result<Vec<String>, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => match storage_val.value {
ValueType::List(list) => {
if list.is_empty() {
return Ok(Vec::new());
}
let len = list.len() as i64;
let start_idx = if start < 0 {
std::cmp::max(0, len + start)
} else {
std::cmp::min(start, len)
};
let stop_idx = if stop < 0 {
std::cmp::max(-1, len + stop)
} else {
std::cmp::min(stop, len - 1)
};
if start_idx > stop_idx || start_idx >= len {
return Ok(Vec::new());
}
let start_usize = start_idx as usize;
let stop_usize = (stop_idx + 1) as usize;
Ok(list[start_usize..std::cmp::min(stop_usize, list.len())].to_vec())
}
_ => Ok(Vec::new())
}
None => Ok(Vec::new())
}
}
fn ltrim(&self, key: &str, start: i64, stop: i64) -> Result<(), DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(())
};
let list = match &mut storage_val.value {
ValueType::List(l) => l,
_ => return Ok(())
};
if list.is_empty() {
return Ok(());
}
let len = list.len() as i64;
let start_idx = if start < 0 {
std::cmp::max(0, len + start)
} else {
std::cmp::min(start, len)
};
let stop_idx = if stop < 0 {
std::cmp::max(-1, len + stop)
} else {
std::cmp::min(stop, len - 1)
};
if start_idx > stop_idx || start_idx >= len {
self.del(key.to_string())?;
} else {
let start_usize = start_idx as usize;
let stop_usize = (stop_idx + 1) as usize;
*list = list[start_usize..std::cmp::min(stop_usize, list.len())].to_vec();
if list.is_empty() {
self.del(key.to_string())?;
} else {
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
}
}
Ok(())
}
fn lrem(&self, key: &str, count: i64, element: &str) -> Result<i64, DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(0)
};
let list = match &mut storage_val.value {
ValueType::List(l) => l,
_ => return Ok(0)
};
let mut removed = 0i64;
if count == 0 {
// Remove all occurrences
let original_len = list.len();
list.retain(|x| x != element);
removed = (original_len - list.len()) as i64;
} else if count > 0 {
// Remove first count occurrences
let mut to_remove = count as usize;
list.retain(|x| {
if x == element && to_remove > 0 {
to_remove -= 1;
removed += 1;
false
} else {
true
}
});
} else {
// Remove last |count| occurrences
let mut to_remove = (-count) as usize;
for i in (0..list.len()).rev() {
if list[i] == element && to_remove > 0 {
list.remove(i);
to_remove -= 1;
removed += 1;
}
}
}
if list.is_empty() {
self.del(key.to_string())?;
} else {
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
}
Ok(removed)
}
// Expiration
fn ttl(&self, key: &str) -> Result<i64, DBError> {
match self.get_storage_value(key)? {
Some(storage_val) => {
if let Some(expires_at) = storage_val.expires_at {
let now = Self::now_millis();
if now >= expires_at {
Ok(-2) // Key has expired
} else {
Ok(((expires_at - now) / 1000) as i64) // TTL in seconds
}
} else {
Ok(-1) // Key exists but has no expiration
}
}
None => Ok(-2) // Key does not exist
}
}
fn expire_seconds(&self, key: &str, secs: u64) -> Result<bool, DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(false)
};
storage_val.expires_at = Some(Self::now_millis() + (secs as u128) * 1000);
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(true)
}
fn pexpire_millis(&self, key: &str, ms: u128) -> Result<bool, DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(false)
};
storage_val.expires_at = Some(Self::now_millis() + ms);
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(true)
}
fn persist(&self, key: &str) -> Result<bool, DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(false)
};
if storage_val.expires_at.is_some() {
storage_val.expires_at = None;
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(true)
} else {
Ok(false)
}
}
fn expire_at_seconds(&self, key: &str, ts_secs: i64) -> Result<bool, DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(false)
};
let expires_at_ms: u128 = if ts_secs <= 0 { 0 } else { (ts_secs as u128) * 1000 };
storage_val.expires_at = Some(expires_at_ms);
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(true)
}
fn pexpire_at_millis(&self, key: &str, ts_ms: i64) -> Result<bool, DBError> {
let mut storage_val = match self.get_storage_value(key)? {
Some(sv) => sv,
None => return Ok(false)
};
let expires_at_ms: u128 = if ts_ms <= 0 { 0 } else { ts_ms as u128 };
storage_val.expires_at = Some(expires_at_ms);
self.set_storage_value(key, storage_val)?;
self.db.flush().map_err(|e| DBError(e.to_string()))?;
Ok(true)
}
fn is_encrypted(&self) -> bool {
self.crypto.is_some()
}
fn clone_arc(&self) -> Arc<dyn StorageBackend> {
// Note: This is a simplified clone - in production you might want to
// handle this differently as sled::Db is already Arc internally
Arc::new(SledStorage {
db: self.db.clone(),
types: self.types.clone(),
crypto: self.crypto.clone(),
})
}
}

57
herodb/src/storage_trait.rs Normal file
View File

@@ -0,0 +1,57 @@
// src/storage_trait.rs
use crate::error::DBError;
use std::sync::Arc;
pub trait StorageBackend: Send + Sync {
// Basic key operations
fn get(&self, key: &str) -> Result<Option<String>, DBError>;
fn set(&self, key: String, value: String) -> Result<(), DBError>;
fn setx(&self, key: String, value: String, expire_ms: u128) -> Result<(), DBError>;
fn del(&self, key: String) -> Result<(), DBError>;
fn exists(&self, key: &str) -> Result<bool, DBError>;
fn keys(&self, pattern: &str) -> Result<Vec<String>, DBError>;
fn dbsize(&self) -> Result<i64, DBError>;
fn flushdb(&self) -> Result<(), DBError>;
fn get_key_type(&self, key: &str) -> Result<Option<String>, DBError>;
// Scanning
fn scan(&self, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError>;
fn hscan(&self, key: &str, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError>;
// Hash operations
fn hset(&self, key: &str, pairs: Vec<(String, String)>) -> Result<i64, DBError>;
fn hget(&self, key: &str, field: &str) -> Result<Option<String>, DBError>;
fn hgetall(&self, key: &str) -> Result<Vec<(String, String)>, DBError>;
fn hdel(&self, key: &str, fields: Vec<String>) -> Result<i64, DBError>;
fn hexists(&self, key: &str, field: &str) -> Result<bool, DBError>;
fn hkeys(&self, key: &str) -> Result<Vec<String>, DBError>;
fn hvals(&self, key: &str) -> Result<Vec<String>, DBError>;
fn hlen(&self, key: &str) -> Result<i64, DBError>;
fn hmget(&self, key: &str, fields: Vec<String>) -> Result<Vec<Option<String>>, DBError>;
fn hsetnx(&self, key: &str, field: &str, value: &str) -> Result<bool, DBError>;
// List operations
fn lpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError>;
fn rpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError>;
fn lpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError>;
fn rpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError>;
fn llen(&self, key: &str) -> Result<i64, DBError>;
fn lindex(&self, key: &str, index: i64) -> Result<Option<String>, DBError>;
fn lrange(&self, key: &str, start: i64, stop: i64) -> Result<Vec<String>, DBError>;
fn ltrim(&self, key: &str, start: i64, stop: i64) -> Result<(), DBError>;
fn lrem(&self, key: &str, count: i64, element: &str) -> Result<i64, DBError>;
// Expiration
fn ttl(&self, key: &str) -> Result<i64, DBError>;
fn expire_seconds(&self, key: &str, secs: u64) -> Result<bool, DBError>;
fn pexpire_millis(&self, key: &str, ms: u128) -> Result<bool, DBError>;
fn persist(&self, key: &str) -> Result<bool, DBError>;
fn expire_at_seconds(&self, key: &str, ts_secs: i64) -> Result<bool, DBError>;
fn pexpire_at_millis(&self, key: &str, ts_ms: i64) -> Result<bool, DBError>;
// Metadata
fn is_encrypted(&self) -> bool;
// Clone to Arc for sharing
fn clone_arc(&self) -> Arc<dyn StorageBackend>;
}