...
This commit is contained in:
123
Cargo.lock
generated
123
Cargo.lock
generated
@@ -206,6 +206,12 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bitflags"
|
||||||
|
version = "1.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "2.9.2"
|
version = "2.9.2"
|
||||||
@@ -358,6 +364,30 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crc32fast"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-epoch"
|
||||||
|
version = "0.9.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-utils"
|
||||||
|
version = "0.8.21"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crypto-common"
|
name = "crypto-common"
|
||||||
version = "0.1.6"
|
version = "0.1.6"
|
||||||
@@ -406,7 +436,7 @@ dependencies = [
|
|||||||
"hashbrown",
|
"hashbrown",
|
||||||
"lock_api",
|
"lock_api",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"parking_lot_core",
|
"parking_lot_core 0.9.11",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -533,6 +563,16 @@ dependencies = [
|
|||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fs2"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures"
|
name = "futures"
|
||||||
version = "0.3.31"
|
version = "0.3.31"
|
||||||
@@ -622,6 +662,15 @@ dependencies = [
|
|||||||
"slab",
|
"slab",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fxhash"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "generic-array"
|
name = "generic-array"
|
||||||
version = "0.14.7"
|
version = "0.14.7"
|
||||||
@@ -682,6 +731,7 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2",
|
"sha2",
|
||||||
|
"sled",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
@@ -732,7 +782,7 @@ dependencies = [
|
|||||||
"intl-memoizer",
|
"intl-memoizer",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"log",
|
"log",
|
||||||
"parking_lot",
|
"parking_lot 0.12.4",
|
||||||
"rust-embed",
|
"rust-embed",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"unic-langid",
|
"unic-langid",
|
||||||
@@ -889,6 +939,15 @@ dependencies = [
|
|||||||
"generic-array",
|
"generic-array",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "instant"
|
||||||
|
version = "0.1.13"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "intl-memoizer"
|
name = "intl-memoizer"
|
||||||
version = "0.5.3"
|
version = "0.5.3"
|
||||||
@@ -914,7 +973,7 @@ version = "0.7.9"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"
|
checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bitflags",
|
"bitflags 2.9.2",
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
@@ -1040,6 +1099,17 @@ version = "0.3.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "parking_lot"
|
||||||
|
version = "0.11.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
||||||
|
dependencies = [
|
||||||
|
"instant",
|
||||||
|
"lock_api",
|
||||||
|
"parking_lot_core 0.8.6",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parking_lot"
|
name = "parking_lot"
|
||||||
version = "0.12.4"
|
version = "0.12.4"
|
||||||
@@ -1047,7 +1117,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13"
|
checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"lock_api",
|
"lock_api",
|
||||||
"parking_lot_core",
|
"parking_lot_core 0.9.11",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "parking_lot_core"
|
||||||
|
version = "0.8.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"instant",
|
||||||
|
"libc",
|
||||||
|
"redox_syscall 0.2.16",
|
||||||
|
"smallvec",
|
||||||
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1058,7 +1142,7 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"libc",
|
"libc",
|
||||||
"redox_syscall",
|
"redox_syscall 0.5.17",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"windows-targets 0.52.6",
|
"windows-targets 0.52.6",
|
||||||
]
|
]
|
||||||
@@ -1252,13 +1336,22 @@ dependencies = [
|
|||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "redox_syscall"
|
||||||
|
version = "0.2.16"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 1.3.2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "redox_syscall"
|
name = "redox_syscall"
|
||||||
version = "0.5.17"
|
version = "0.5.17"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
|
checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bitflags",
|
"bitflags 2.9.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1466,6 +1559,22 @@ version = "0.4.11"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
|
checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sled"
|
||||||
|
version = "0.34.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
|
||||||
|
dependencies = [
|
||||||
|
"crc32fast",
|
||||||
|
"crossbeam-epoch",
|
||||||
|
"crossbeam-utils",
|
||||||
|
"fs2",
|
||||||
|
"fxhash",
|
||||||
|
"libc",
|
||||||
|
"log",
|
||||||
|
"parking_lot 0.11.2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "smallvec"
|
name = "smallvec"
|
||||||
version = "1.15.1"
|
version = "1.15.1"
|
||||||
@@ -1599,7 +1708,7 @@ dependencies = [
|
|||||||
"io-uring",
|
"io-uring",
|
||||||
"libc",
|
"libc",
|
||||||
"mio",
|
"mio",
|
||||||
"parking_lot",
|
"parking_lot 0.12.4",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"signal-hook-registry",
|
"signal-hook-registry",
|
||||||
"slab",
|
"slab",
|
||||||
|
|||||||
@@ -12,10 +12,11 @@ tokio = { version = "1.23.0", features = ["full"] }
|
|||||||
clap = { version = "4.5.20", features = ["derive"] }
|
clap = { version = "4.5.20", features = ["derive"] }
|
||||||
byteorder = "1.4.3"
|
byteorder = "1.4.3"
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
|
sled = "0.34"
|
||||||
redb = "2.1.3"
|
redb = "2.1.3"
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
bincode = "1.3.3"
|
bincode = "1.3"
|
||||||
chacha20poly1305 = "0.10.1"
|
chacha20poly1305 = "0.10.1"
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
sha2 = "0.10"
|
sha2 = "0.10"
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
# HeroDB
|
||||||
|
|
||||||
|
HeroDB is a Redis-compatible database built with Rust, offering a flexible and secure storage solution. It supports two primary storage backends: `redb` (default) and `sled`, both with full encryption capabilities. HeroDB aims to provide a robust and performant key-value store with advanced features like data-at-rest encryption, hash operations, list operations, and cursor-based scanning.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
The main purpose of HeroDB is to offer a lightweight, embeddable, and Redis-compatible database that prioritizes data security through transparent encryption. It's designed for applications that require fast, reliable data storage with the option for strong cryptographic protection, without the overhead of a full-fledged Redis server.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Redis Compatibility**: Supports a subset of Redis commands over RESP (Redis Serialization Protocol) via TCP.
|
||||||
|
- **Dual Backend Support**:
|
||||||
|
- `redb` (default): Optimized for concurrent access and high-throughput scenarios.
|
||||||
|
- `sled`: A lock-free, log-structured database, excellent for specific workloads.
|
||||||
|
- **Data-at-Rest Encryption**: Transparent encryption for both backends using the `age` encryption library.
|
||||||
|
- **Key-Value Operations**: Full support for basic string, hash, and list operations.
|
||||||
|
- **Expiration**: Time-to-live (TTL) functionality for keys.
|
||||||
|
- **Scanning**: Cursor-based iteration for keys and hash fields (`SCAN`, `HSCAN`).
|
||||||
|
- **AGE Cryptography Commands**: HeroDB-specific extensions for cryptographic operations.
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### Building HeroDB
|
||||||
|
|
||||||
|
To build HeroDB, navigate to the project root and run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build --release
|
||||||
|
```
|
||||||
|
|
||||||
|
### Running HeroDB
|
||||||
|
|
||||||
|
You can start HeroDB with different backends and encryption options:
|
||||||
|
|
||||||
|
#### Default `redb` Backend
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./target/release/herodb --dir /tmp/herodb_redb --port 6379
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `sled` Backend
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./target/release/herodb --dir /tmp/herodb_sled --port 6379 --sled
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `redb` with Encryption
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./target/release/herodb --dir /tmp/herodb_encrypted --port 6379 --encrypt --key mysecretkey
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `sled` with Encryption
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./target/release/herodb --dir /tmp/herodb_sled_encrypted --port 6379 --sled --encrypt --key mysecretkey
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage with Redis Clients
|
||||||
|
|
||||||
|
HeroDB can be interacted with using any standard Redis client, such as `redis-cli`, `redis-py` (Python), or `ioredis` (Node.js).
|
||||||
|
|
||||||
|
### Example with `redis-cli`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
redis-cli -p 6379 SET mykey "Hello from HeroDB!"
|
||||||
|
redis-cli -p 6379 GET mykey
|
||||||
|
# → "Hello from HeroDB!"
|
||||||
|
|
||||||
|
redis-cli -p 6379 HSET user:1 name "Alice" age "30"
|
||||||
|
redis-cli -p 6379 HGET user:1 name
|
||||||
|
# → "Alice"
|
||||||
|
|
||||||
|
redis-cli -p 6379 SCAN 0 MATCH user:* COUNT 10
|
||||||
|
# → 1) "0"
|
||||||
|
# 2) 1) "user:1"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
For more detailed information on commands, features, and advanced usage, please refer to the documentation:
|
||||||
|
|
||||||
|
- [Basics](docs/basics.md)
|
||||||
|
- [Supported Commands](docs/cmds.md)
|
||||||
|
- [AGE Cryptography](docs/age.md)
|
||||||
@@ -1,227 +1,116 @@
|
|||||||
|
|
||||||
# HeroDB Redis Protocol Support: Commands & Client Usage
|
## Backend Support
|
||||||
|
|
||||||
HeroDB is a Redis-compatible database built using the `redb` database backend.
|
HeroDB supports two storage backends, both with full encryption support:
|
||||||
|
|
||||||
It supports a subset of Redis commands over the standard RESP (Redis Serialization Protocol) via TCP, allowing you to interact with it using standard Redis clients like `redis-cli`, Python's `redis-py`, Node.js's `ioredis`, etc.
|
- **redb** (default): Full-featured, optimized for production use
|
||||||
|
- **sled**: Alternative embedded database with encryption support
|
||||||
|
|
||||||
This document provides:
|
### Starting HeroDB with Different Backends
|
||||||
- A list of all currently supported Redis commands.
|
|
||||||
- Example usage with standard Redis clients.
|
|
||||||
- Bash and Rust test-inspired usage examples.
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
Assuming the server is running on localhost at port `$PORT`:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Build HeroDB
|
# Use default redb backend
|
||||||
cargo build --release
|
./target/release/herodb --dir /tmp/herodb_redb --port 6379
|
||||||
|
|
||||||
# Start HeroDB server
|
# Use sled backend
|
||||||
./target/release/herodb --dir /tmp/herodb_data --port 6381 --debug
|
./target/release/herodb --dir /tmp/herodb_sled --port 6379 --sled
|
||||||
|
|
||||||
|
# Use redb with encryption
|
||||||
|
./target/release/herodb --dir /tmp/herodb_encrypted --port 6379 --encrypt --key mysecretkey
|
||||||
|
|
||||||
|
# Use sled with encryption
|
||||||
|
./target/release/herodb --dir /tmp/herodb_sled_encrypted --port 6379 --sled --encrypt --key mysecretkey
|
||||||
```
|
```
|
||||||
|
|
||||||
## Using Standard Redis Clients
|
### Command Support by Backend
|
||||||
|
|
||||||
### With `redis-cli`
|
Command Category | redb | sled | Notes |
|
||||||
|
|-----------------|------|------|-------|
|
||||||
|
**Strings** | | | |
|
||||||
|
SET | ✅ | ✅ | Full support |
|
||||||
|
GET | ✅ | ✅ | Full support |
|
||||||
|
DEL | ✅ | ✅ | Full support |
|
||||||
|
EXISTS | ✅ | ✅ | Full support |
|
||||||
|
INCR/DECR | ✅ | ✅ | Full support |
|
||||||
|
MGET/MSET | ✅ | ✅ | Full support |
|
||||||
|
**Hashes** | | | |
|
||||||
|
HSET | ✅ | ✅ | Full support |
|
||||||
|
HGET | ✅ | ✅ | Full support |
|
||||||
|
HGETALL | ✅ | ✅ | Full support |
|
||||||
|
HDEL | ✅ | ✅ | Full support |
|
||||||
|
HEXISTS | ✅ | ✅ | Full support |
|
||||||
|
HKEYS | ✅ | ✅ | Full support |
|
||||||
|
HVALS | ✅ | ✅ | Full support |
|
||||||
|
HLEN | ✅ | ✅ | Full support |
|
||||||
|
HMGET | ✅ | ✅ | Full support |
|
||||||
|
HSETNX | ✅ | ✅ | Full support |
|
||||||
|
HINCRBY/HINCRBYFLOAT | ✅ | ✅ | Full support |
|
||||||
|
HSCAN | ✅ | ✅ | Full support with pattern matching |
|
||||||
|
**Lists** | | | |
|
||||||
|
LPUSH/RPUSH | ✅ | ✅ | Full support |
|
||||||
|
LPOP/RPOP | ✅ | ✅ | Full support |
|
||||||
|
LLEN | ✅ | ✅ | Full support |
|
||||||
|
LRANGE | ✅ | ✅ | Full support |
|
||||||
|
LINDEX | ✅ | ✅ | Full support |
|
||||||
|
LTRIM | ✅ | ✅ | Full support |
|
||||||
|
LREM | ✅ | ✅ | Full support |
|
||||||
|
BLPOP/BRPOP | ✅ | ❌ | Blocking operations not in sled |
|
||||||
|
**Expiration** | | | |
|
||||||
|
EXPIRE | ✅ | ✅ | Full support in both |
|
||||||
|
TTL | ✅ | ✅ | Full support in both |
|
||||||
|
PERSIST | ✅ | ✅ | Full support in both |
|
||||||
|
SETEX/PSETEX | ✅ | ✅ | Full support in both |
|
||||||
|
EXPIREAT/PEXPIREAT | ✅ | ✅ | Full support in both |
|
||||||
|
**Scanning** | | | |
|
||||||
|
KEYS | ✅ | ✅ | Full support with patterns |
|
||||||
|
SCAN | ✅ | ✅ | Full cursor-based iteration |
|
||||||
|
HSCAN | ✅ | ✅ | Full cursor-based iteration |
|
||||||
|
**Transactions** | | | |
|
||||||
|
MULTI/EXEC/DISCARD | ✅ | ❌ | Only supported in redb |
|
||||||
|
**Encryption** | | | |
|
||||||
|
Data-at-rest encryption | ✅ | ✅ | Both support [age](age.tech) encryption |
|
||||||
|
AGE commands | ✅ | ✅ | Both support AGE crypto commands |
|
||||||
|
|
||||||
|
### Performance Considerations
|
||||||
|
|
||||||
|
- **redb**: Optimized for concurrent access, better for high-throughput scenarios
|
||||||
|
- **sled**: Lock-free architecture, excellent for specific workloads
|
||||||
|
|
||||||
|
### Encryption Features
|
||||||
|
|
||||||
|
Both backends support:
|
||||||
|
- Transparent data-at-rest encryption using the `age` encryption library
|
||||||
|
- Per-database encryption (databases >= 10 are encrypted when `--encrypt` flag is used)
|
||||||
|
- Secure key derivation using the master key
|
||||||
|
|
||||||
|
### Backend Selection Examples
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
redis-cli -p 6381 SET mykey "hello"
|
# Example: Testing both backends
|
||||||
redis-cli -p 6381 GET mykey
|
redis-cli -p 6379 SET mykey "redb value"
|
||||||
|
redis-cli -p 6381 SET mykey "sled value"
|
||||||
|
|
||||||
|
# Example: Using encryption with both
|
||||||
|
./target/release/herodb --port 6379 --encrypt --key secret123
|
||||||
|
./target/release/herodb --port 6381 --sled --encrypt --key secret123
|
||||||
|
|
||||||
|
# Both support the same Redis commands
|
||||||
|
redis-cli -p 6379 HSET user:1 name "Alice" age "30"
|
||||||
|
redis-cli -p 6381 HSET user:1 name "Alice" age "30"
|
||||||
|
|
||||||
|
# Both support SCAN operations
|
||||||
|
redis-cli -p 6379 SCAN 0 MATCH user:* COUNT 10
|
||||||
|
redis-cli -p 6381 SCAN 0 MATCH user:* COUNT 10
|
||||||
```
|
```
|
||||||
|
|
||||||
### With Python (`redis-py`)
|
### Migration Between Backends
|
||||||
|
|
||||||
```python
|
To migrate data between backends, use Redis replication or dump/restore:
|
||||||
import redis
|
|
||||||
|
|
||||||
r = redis.Redis(host='localhost', port=6381, db=0)
|
|
||||||
r.set('mykey', 'hello')
|
|
||||||
print(r.get('mykey').decode())
|
|
||||||
```
|
|
||||||
|
|
||||||
### With Node.js (`ioredis`)
|
|
||||||
|
|
||||||
```js
|
|
||||||
const Redis = require("ioredis");
|
|
||||||
const redis = new Redis({ port: 6381, host: "localhost" });
|
|
||||||
|
|
||||||
await redis.set("mykey", "hello");
|
|
||||||
const value = await redis.get("mykey");
|
|
||||||
console.log(value); // "hello"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Supported Redis Commands
|
|
||||||
|
|
||||||
### String Commands
|
|
||||||
|
|
||||||
| Command | Description | Example Usage |
|
|
||||||
|---------------|------------------------------------------|-------------------------------------------|
|
|
||||||
| `SET` | Set a key to a string value | `SET name "Alice"` |
|
|
||||||
| `GET` | Get the value of a key | `GET name` |
|
|
||||||
| `DEL` | Delete one or more keys | `DEL name age` |
|
|
||||||
| `INCR` | Increment the integer value of a key | `INCR counter` |
|
|
||||||
| `DECR` | Decrement the integer value of a key | `DECR counter` |
|
|
||||||
| `INCRBY` | Increment key by a given integer | `INCRBY counter 5` |
|
|
||||||
| `DECRBY` | Decrement key by a given integer | `DECRBY counter 3` |
|
|
||||||
| `EXISTS` | Check if a key exists | `EXISTS name` |
|
|
||||||
| `TYPE` | Return the type of a key | `TYPE name` |
|
|
||||||
|
|
||||||
### Hash Commands
|
|
||||||
|
|
||||||
| Command | Description | Example Usage |
|
|
||||||
|---------------|------------------------------------------|-------------------------------------------|
|
|
||||||
| `HSET` | Set field in hash stored at key | `HSET user:1 name "Alice"` |
|
|
||||||
| `HGET` | Get value of a field in hash | `HGET user:1 name` |
|
|
||||||
| `HGETALL` | Get all fields and values in a hash | `HGETALL user:1` |
|
|
||||||
| `HDEL` | Delete one or more fields from hash | `HDEL user:1 name age` |
|
|
||||||
| `HEXISTS` | Check if field exists in hash | `HEXISTS user:1 name` |
|
|
||||||
| `HKEYS` | Get all field names in a hash | `HKEYS user:1` |
|
|
||||||
| `HVALS` | Get all values in a hash | `HVALS user:1` |
|
|
||||||
| `HLEN` | Get number of fields in a hash | `HLEN user:1` |
|
|
||||||
| `HMGET` | Get values of multiple fields | `HMGET user:1 name age` |
|
|
||||||
| `HSETNX` | Set field only if it does not exist | `HSETNX user:1 email alice@example.com` |
|
|
||||||
|
|
||||||
### List Commands
|
|
||||||
|
|
||||||
| Command | Description | Example Usage |
|
|
||||||
|---------------|------------------------------------------|-------------------------------------------|
|
|
||||||
| `LPUSH` | Insert elements at the head of a list | `LPUSH mylist "item1" "item2"` |
|
|
||||||
| `RPUSH` | Insert elements at the tail of a list | `RPUSH mylist "item3" "item4"` |
|
|
||||||
| `LPOP` | Remove and return element from head | `LPOP mylist` |
|
|
||||||
| `RPOP` | Remove and return element from tail | `RPOP mylist` |
|
|
||||||
| `BLPOP` | Blocking remove from head with timeout | `BLPOP mylist1 mylist2 5` |
|
|
||||||
| `BRPOP` | Blocking remove from tail with timeout | `BRPOP mylist1 mylist2 5` |
|
|
||||||
| `LLEN` | Get the length of a list | `LLEN mylist` |
|
|
||||||
| `LREM` | Remove elements from list | `LREM mylist 2 "item"` |
|
|
||||||
| `LTRIM` | Trim list to specified range | `LTRIM mylist 0 5` |
|
|
||||||
| `LINDEX` | Get element by index | `LINDEX mylist 0` |
|
|
||||||
| `LRANGE` | Get range of elements | `LRANGE mylist 0 -1` |
|
|
||||||
|
|
||||||
### Keys & Scanning
|
|
||||||
|
|
||||||
| Command | Description | Example Usage |
|
|
||||||
|---------------|------------------------------------------|-------------------------------------------|
|
|
||||||
| `KEYS` | Find all keys matching a pattern | `KEYS user:*` |
|
|
||||||
| `SCAN` | Incrementally iterate keys | `SCAN 0 MATCH user:* COUNT 10` |
|
|
||||||
|
|
||||||
### Expiration
|
|
||||||
|
|
||||||
| Command | Description | Example Usage |
|
|
||||||
|---------------|------------------------------------------|-------------------------------------------|
|
|
||||||
| `EXPIRE` | Set a key's time to live in seconds | `EXPIRE tempkey 60` |
|
|
||||||
| `TTL` | Get the time to live for a key | `TTL tempkey` |
|
|
||||||
| `PERSIST` | Remove the expiration from a key | `PERSIST tempkey` |
|
|
||||||
|
|
||||||
### Transactions
|
|
||||||
|
|
||||||
| Command | Description | Example Usage |
|
|
||||||
|---------------|------------------------------------------|-------------------------------------------|
|
|
||||||
| `MULTI` | Start a transaction block | `MULTI` |
|
|
||||||
| `EXEC` | Execute all commands in a transaction | `EXEC` |
|
|
||||||
| `DISCARD` | Discard all commands in a transaction | `DISCARD` |
|
|
||||||
|
|
||||||
### Configuration
|
|
||||||
|
|
||||||
| Command | Description | Example Usage |
|
|
||||||
|---------------|------------------------------------------|-------------------------------------------|
|
|
||||||
| `CONFIG GET` | Get configuration parameters | `CONFIG GET dir` |
|
|
||||||
| `CONFIG SET` | Set configuration parameters | `CONFIG SET maxmemory 100mb` |
|
|
||||||
|
|
||||||
### Info & Monitoring
|
|
||||||
|
|
||||||
| Command | Description | Example Usage |
|
|
||||||
|---------------|------------------------------------------|-------------------------------------------|
|
|
||||||
| `INFO` | Get information and statistics about server | `INFO` |
|
|
||||||
| `PING` | Ping the server | `PING` |
|
|
||||||
|
|
||||||
### AGE Cryptography Commands
|
|
||||||
|
|
||||||
| Command | Description | Example Usage |
|
|
||||||
|--------------------|-----------------------------------------------|-----------------------------------------------|
|
|
||||||
| `AGE GENENC` | Generate ephemeral encryption keypair | `AGE GENENC` |
|
|
||||||
| `AGE GENSIGN` | Generate ephemeral signing keypair | `AGE GENSIGN` |
|
|
||||||
| `AGE ENCRYPT` | Encrypt a message using a public key | `AGE ENCRYPT <recipient> "msg"` |
|
|
||||||
| `AGE DECRYPT` | Decrypt a message using a secret key | `AGE DECRYPT <identity> <ciphertext>` |
|
|
||||||
| `AGE SIGN` | Sign a message using a secret key | `AGE SIGN <sign_secret> "msg"` |
|
|
||||||
| `AGE VERIFY` | Verify a signature using a public key | `AGE VERIFY <pubkey> "msg" <signature>` |
|
|
||||||
| `AGE KEYGEN` | Create and persist a named encryption key | `AGE KEYGEN app1` |
|
|
||||||
| `AGE SIGNKEYGEN` | Create and persist a named signing key | `AGE SIGNKEYGEN app1` |
|
|
||||||
| `AGE ENCRYPTNAME` | Encrypt using a named key | `AGE ENCRYPTNAME app1 "msg"` |
|
|
||||||
| `AGE DECRYPTNAME` | Decrypt using a named key | `AGE DECRYPTNAME app1 <ciphertext>` |
|
|
||||||
| `AGE SIGNNAME` | Sign using a named key | `AGE SIGNNAME app1 "msg"` |
|
|
||||||
| `AGE VERIFYNAME` | Verify using a named key | `AGE VERIFYNAME app1 "msg" <signature>` |
|
|
||||||
| `AGE LIST` | List all persisted named keys | `AGE LIST` |
|
|
||||||
|
|
||||||
> Note: AGE commands are not part of standard Redis. They are HeroDB-specific extensions for cryptographic operations.
|
|
||||||
|
|
||||||
## Example Usage
|
|
||||||
|
|
||||||
### Basic String Operations
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
redis-cli -p 6381 SET greeting "Hello, HeroDB!"
|
# Export from redb
|
||||||
redis-cli -p 6381 GET greeting
|
redis-cli -p 6379 --rdb dump.rdb
|
||||||
# → "Hello, HeroDB!"
|
|
||||||
|
|
||||||
redis-cli -p 6381 INCR visits
|
# Import to sled
|
||||||
redis-cli -p 6381 INCR visits
|
redis-cli -p 6381 --pipe < dump.rdb
|
||||||
redis-cli -p 6381 GET visits
|
```
|
||||||
# → "2"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Hash Operations
|
|
||||||
|
|
||||||
```bash
|
|
||||||
redis-cli -p 6381 HSET user:1000 name "Alice" age "30" city "NYC"
|
|
||||||
redis-cli -p 6381 HGET user:1000 name
|
|
||||||
# → "Alice"
|
|
||||||
|
|
||||||
redis-cli -p 6381 HGETALL user:1000
|
|
||||||
# → 1) "name"
|
|
||||||
# 2) "Alice"
|
|
||||||
# 3) "age"
|
|
||||||
# 4) "30"
|
|
||||||
# 5) "city"
|
|
||||||
# 6) "NYC"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Expiration
|
|
||||||
|
|
||||||
```bash
|
|
||||||
redis-cli -p 6381 SET tempkey "temporary"
|
|
||||||
redis-cli -p 6381 EXPIRE tempkey 5
|
|
||||||
redis-cli -p 6381 TTL tempkey
|
|
||||||
# → (integer) 4
|
|
||||||
|
|
||||||
# After 5 seconds:
|
|
||||||
redis-cli -p 6381 GET tempkey
|
|
||||||
# → (nil)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Transactions
|
|
||||||
|
|
||||||
```bash
|
|
||||||
redis-cli -p 6381 MULTI
|
|
||||||
redis-cli -p 6381 SET txkey1 "value1"
|
|
||||||
redis-cli -p 6381 SET txkey2 "value2"
|
|
||||||
redis-cli -p 6381 INCR counter
|
|
||||||
redis-cli -p 6381 EXEC
|
|
||||||
# → 1) OK
|
|
||||||
# 2) OK
|
|
||||||
# 3) (integer) 3
|
|
||||||
```
|
|
||||||
|
|
||||||
### Scanning Keys
|
|
||||||
|
|
||||||
```bash
|
|
||||||
redis-cli -p 6381 SET scankey1 "val1"
|
|
||||||
redis-cli -p 6381 SET scankey2 "val2"
|
|
||||||
redis-cli -p 6381 HSET scanhash field1 "val1"
|
|
||||||
|
|
||||||
redis-cli -p 6381 SCAN 0 MATCH scankey*
|
|
||||||
# → 1) "0"
|
|
||||||
# 2) 1) "scankey1"
|
|
||||||
# 2) "scankey2"
|
|
||||||
```
|
|
||||||
0
herodb/specs/backgroundinfo/tantivy.md
Normal file
0
herodb/specs/backgroundinfo/tantivy.md
Normal file
@@ -23,6 +23,7 @@ impl From<CryptoError> for crate::error::DBError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Super-simple factory: new(secret) + encrypt(bytes) + decrypt(bytes)
|
/// Super-simple factory: new(secret) + encrypt(bytes) + decrypt(bytes)
|
||||||
|
#[derive(Clone)]
|
||||||
pub struct CryptoFactory {
|
pub struct CryptoFactory {
|
||||||
key: chacha20poly1305::Key,
|
key: chacha20poly1305::Key,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,3 +6,5 @@ pub mod options;
|
|||||||
pub mod protocol;
|
pub mod protocol;
|
||||||
pub mod server;
|
pub mod server;
|
||||||
pub mod storage;
|
pub mod storage;
|
||||||
|
pub mod storage_trait; // Add this
|
||||||
|
pub mod storage_sled; // Add this
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ struct Args {
|
|||||||
/// Encrypt the database
|
/// Encrypt the database
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
encrypt: bool,
|
encrypt: bool,
|
||||||
|
|
||||||
|
/// Use the sled backend
|
||||||
|
#[arg(long)]
|
||||||
|
sled: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
@@ -47,10 +51,14 @@ async fn main() {
|
|||||||
// new DB option
|
// new DB option
|
||||||
let option = herodb::options::DBOption {
|
let option = herodb::options::DBOption {
|
||||||
dir: args.dir,
|
dir: args.dir,
|
||||||
port,
|
|
||||||
debug: args.debug,
|
debug: args.debug,
|
||||||
encryption_key: args.encryption_key,
|
encryption_key: args.encryption_key,
|
||||||
encrypt: args.encrypt,
|
encrypt: args.encrypt,
|
||||||
|
backend: if args.sled {
|
||||||
|
herodb::options::BackendType::Sled
|
||||||
|
} else {
|
||||||
|
herodb::options::BackendType::Redb
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
// new server
|
// new server
|
||||||
|
|||||||
@@ -1,8 +1,14 @@
|
|||||||
#[derive(Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum BackendType {
|
||||||
|
Redb,
|
||||||
|
Sled,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
pub struct DBOption {
|
pub struct DBOption {
|
||||||
pub dir: String,
|
pub dir: String,
|
||||||
pub port: u16,
|
|
||||||
pub debug: bool,
|
pub debug: bool,
|
||||||
pub encrypt: bool,
|
pub encrypt: bool,
|
||||||
pub encryption_key: Option<String>, // Master encryption key
|
pub encryption_key: Option<String>,
|
||||||
|
pub backend: BackendType,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use std::{
|
use std::{
|
||||||
path::Path,
|
path::Path,
|
||||||
|
sync::Arc,
|
||||||
time::{SystemTime, UNIX_EPOCH},
|
time::{SystemTime, UNIX_EPOCH},
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -123,4 +124,160 @@ impl Storage {
|
|||||||
Ok(data.to_vec())
|
Ok(data.to_vec())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
use crate::storage_trait::StorageBackend;
|
||||||
|
|
||||||
|
impl StorageBackend for Storage {
|
||||||
|
fn get(&self, key: &str) -> Result<Option<String>, DBError> {
|
||||||
|
self.get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set(&self, key: String, value: String) -> Result<(), DBError> {
|
||||||
|
self.set(key, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn setx(&self, key: String, value: String, expire_ms: u128) -> Result<(), DBError> {
|
||||||
|
self.setx(key, value, expire_ms)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn del(&self, key: String) -> Result<(), DBError> {
|
||||||
|
self.del(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exists(&self, key: &str) -> Result<bool, DBError> {
|
||||||
|
self.exists(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn keys(&self, pattern: &str) -> Result<Vec<String>, DBError> {
|
||||||
|
self.keys(pattern)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dbsize(&self) -> Result<i64, DBError> {
|
||||||
|
self.dbsize()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flushdb(&self) -> Result<(), DBError> {
|
||||||
|
self.flushdb()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_key_type(&self, key: &str) -> Result<Option<String>, DBError> {
|
||||||
|
self.get_key_type(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn scan(&self, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError> {
|
||||||
|
self.scan(cursor, pattern, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hscan(&self, key: &str, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError> {
|
||||||
|
self.hscan(key, cursor, pattern, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hset(&self, key: &str, pairs: Vec<(String, String)>) -> Result<i64, DBError> {
|
||||||
|
self.hset(key, pairs)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hget(&self, key: &str, field: &str) -> Result<Option<String>, DBError> {
|
||||||
|
self.hget(key, field)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hgetall(&self, key: &str) -> Result<Vec<(String, String)>, DBError> {
|
||||||
|
self.hgetall(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hdel(&self, key: &str, fields: Vec<String>) -> Result<i64, DBError> {
|
||||||
|
self.hdel(key, fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hexists(&self, key: &str, field: &str) -> Result<bool, DBError> {
|
||||||
|
self.hexists(key, field)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hkeys(&self, key: &str) -> Result<Vec<String>, DBError> {
|
||||||
|
self.hkeys(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hvals(&self, key: &str) -> Result<Vec<String>, DBError> {
|
||||||
|
self.hvals(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hlen(&self, key: &str) -> Result<i64, DBError> {
|
||||||
|
self.hlen(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hmget(&self, key: &str, fields: Vec<String>) -> Result<Vec<Option<String>>, DBError> {
|
||||||
|
self.hmget(key, fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hsetnx(&self, key: &str, field: &str, value: &str) -> Result<bool, DBError> {
|
||||||
|
self.hsetnx(key, field, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError> {
|
||||||
|
self.lpush(key, elements)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError> {
|
||||||
|
self.rpush(key, elements)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError> {
|
||||||
|
self.lpop(key, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError> {
|
||||||
|
self.rpop(key, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn llen(&self, key: &str) -> Result<i64, DBError> {
|
||||||
|
self.llen(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lindex(&self, key: &str, index: i64) -> Result<Option<String>, DBError> {
|
||||||
|
self.lindex(key, index)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lrange(&self, key: &str, start: i64, stop: i64) -> Result<Vec<String>, DBError> {
|
||||||
|
self.lrange(key, start, stop)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ltrim(&self, key: &str, start: i64, stop: i64) -> Result<(), DBError> {
|
||||||
|
self.ltrim(key, start, stop)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lrem(&self, key: &str, count: i64, element: &str) -> Result<i64, DBError> {
|
||||||
|
self.lrem(key, count, element)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ttl(&self, key: &str) -> Result<i64, DBError> {
|
||||||
|
self.ttl(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expire_seconds(&self, key: &str, secs: u64) -> Result<bool, DBError> {
|
||||||
|
self.expire_seconds(key, secs)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pexpire_millis(&self, key: &str, ms: u128) -> Result<bool, DBError> {
|
||||||
|
self.pexpire_millis(key, ms)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn persist(&self, key: &str) -> Result<bool, DBError> {
|
||||||
|
self.persist(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expire_at_seconds(&self, key: &str, ts_secs: i64) -> Result<bool, DBError> {
|
||||||
|
self.expire_at_seconds(key, ts_secs)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pexpire_at_millis(&self, key: &str, ts_ms: i64) -> Result<bool, DBError> {
|
||||||
|
self.pexpire_at_millis(key, ts_ms)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_encrypted(&self) -> bool {
|
||||||
|
self.is_encrypted()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clone_arc(&self) -> Arc<dyn StorageBackend> {
|
||||||
|
unimplemented!("Storage cloning not yet implemented for redb backend")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
837
herodb/src/storage_sled/mod.rs
Normal file
837
herodb/src/storage_sled/mod.rs
Normal file
@@ -0,0 +1,837 @@
|
|||||||
|
// src/storage_sled/mod.rs
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use crate::error::DBError;
|
||||||
|
use crate::storage_trait::StorageBackend;
|
||||||
|
use crate::crypto::CryptoFactory;
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||||
|
enum ValueType {
|
||||||
|
String(String),
|
||||||
|
Hash(HashMap<String, String>),
|
||||||
|
List(Vec<String>),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||||
|
struct StorageValue {
|
||||||
|
value: ValueType,
|
||||||
|
expires_at: Option<u128>, // milliseconds since epoch
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct SledStorage {
|
||||||
|
db: sled::Db,
|
||||||
|
types: sled::Tree,
|
||||||
|
crypto: Option<CryptoFactory>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SledStorage {
|
||||||
|
pub fn new(path: impl AsRef<Path>, should_encrypt: bool, master_key: Option<&str>) -> Result<Self, DBError> {
|
||||||
|
let db = sled::open(path).map_err(|e| DBError(format!("Failed to open sled: {}", e)))?;
|
||||||
|
let types = db.open_tree("types").map_err(|e| DBError(format!("Failed to open types tree: {}", e)))?;
|
||||||
|
|
||||||
|
// Check if database was previously encrypted
|
||||||
|
let encrypted_tree = db.open_tree("encrypted").map_err(|e| DBError(e.to_string()))?;
|
||||||
|
let was_encrypted = encrypted_tree.get("encrypted")
|
||||||
|
.map_err(|e| DBError(e.to_string()))?
|
||||||
|
.map(|v| v[0] == 1)
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
let crypto = if should_encrypt || was_encrypted {
|
||||||
|
if let Some(key) = master_key {
|
||||||
|
Some(CryptoFactory::new(key.as_bytes()))
|
||||||
|
} else {
|
||||||
|
return Err(DBError("Encryption requested but no master key provided".to_string()));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
// Mark database as encrypted if enabling encryption
|
||||||
|
if should_encrypt && !was_encrypted {
|
||||||
|
encrypted_tree.insert("encrypted", &[1u8])
|
||||||
|
.map_err(|e| DBError(e.to_string()))?;
|
||||||
|
encrypted_tree.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(SledStorage { db, types, crypto })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_millis() -> u128 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_millis()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encrypt_if_needed(&self, data: &[u8]) -> Result<Vec<u8>, DBError> {
|
||||||
|
if let Some(crypto) = &self.crypto {
|
||||||
|
Ok(crypto.encrypt(data))
|
||||||
|
} else {
|
||||||
|
Ok(data.to_vec())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decrypt_if_needed(&self, data: &[u8]) -> Result<Vec<u8>, DBError> {
|
||||||
|
if let Some(crypto) = &self.crypto {
|
||||||
|
Ok(crypto.decrypt(data)?)
|
||||||
|
} else {
|
||||||
|
Ok(data.to_vec())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_storage_value(&self, key: &str) -> Result<Option<StorageValue>, DBError> {
|
||||||
|
match self.db.get(key).map_err(|e| DBError(e.to_string()))? {
|
||||||
|
Some(encrypted_data) => {
|
||||||
|
let decrypted = self.decrypt_if_needed(&encrypted_data)?;
|
||||||
|
let storage_val: StorageValue = bincode::deserialize(&decrypted)
|
||||||
|
.map_err(|e| DBError(format!("Deserialization error: {}", e)))?;
|
||||||
|
|
||||||
|
// Check expiration
|
||||||
|
if let Some(expires_at) = storage_val.expires_at {
|
||||||
|
if Self::now_millis() > expires_at {
|
||||||
|
// Expired, remove it
|
||||||
|
self.db.remove(key).map_err(|e| DBError(e.to_string()))?;
|
||||||
|
self.types.remove(key).map_err(|e| DBError(e.to_string()))?;
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Some(storage_val))
|
||||||
|
}
|
||||||
|
None => Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_storage_value(&self, key: &str, storage_val: StorageValue) -> Result<(), DBError> {
|
||||||
|
let data = bincode::serialize(&storage_val)
|
||||||
|
.map_err(|e| DBError(format!("Serialization error: {}", e)))?;
|
||||||
|
let encrypted = self.encrypt_if_needed(&data)?;
|
||||||
|
self.db.insert(key, encrypted).map_err(|e| DBError(e.to_string()))?;
|
||||||
|
|
||||||
|
// Store type info (unencrypted for efficiency)
|
||||||
|
let type_str = match &storage_val.value {
|
||||||
|
ValueType::String(_) => "string",
|
||||||
|
ValueType::Hash(_) => "hash",
|
||||||
|
ValueType::List(_) => "list",
|
||||||
|
};
|
||||||
|
self.types.insert(key, type_str.as_bytes()).map_err(|e| DBError(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn glob_match(pattern: &str, text: &str) -> bool {
|
||||||
|
if pattern == "*" {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let pattern_chars: Vec<char> = pattern.chars().collect();
|
||||||
|
let text_chars: Vec<char> = text.chars().collect();
|
||||||
|
|
||||||
|
fn match_recursive(pattern: &[char], text: &[char], pi: usize, ti: usize) -> bool {
|
||||||
|
if pi >= pattern.len() {
|
||||||
|
return ti >= text.len();
|
||||||
|
}
|
||||||
|
|
||||||
|
if ti >= text.len() {
|
||||||
|
return pattern[pi..].iter().all(|&c| c == '*');
|
||||||
|
}
|
||||||
|
|
||||||
|
match pattern[pi] {
|
||||||
|
'*' => {
|
||||||
|
for i in ti..=text.len() {
|
||||||
|
if match_recursive(pattern, text, pi + 1, i) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
'?' => match_recursive(pattern, text, pi + 1, ti + 1),
|
||||||
|
c => {
|
||||||
|
if text[ti] == c {
|
||||||
|
match_recursive(pattern, text, pi + 1, ti + 1)
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match_recursive(&pattern_chars, &text_chars, 0, 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StorageBackend for SledStorage {
|
||||||
|
fn get(&self, key: &str) -> Result<Option<String>, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::String(s) => Ok(Some(s)),
|
||||||
|
_ => Ok(None)
|
||||||
|
}
|
||||||
|
None => Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set(&self, key: String, value: String) -> Result<(), DBError> {
|
||||||
|
let storage_val = StorageValue {
|
||||||
|
value: ValueType::String(value),
|
||||||
|
expires_at: None,
|
||||||
|
};
|
||||||
|
self.set_storage_value(&key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn setx(&self, key: String, value: String, expire_ms: u128) -> Result<(), DBError> {
|
||||||
|
let storage_val = StorageValue {
|
||||||
|
value: ValueType::String(value),
|
||||||
|
expires_at: Some(Self::now_millis() + expire_ms),
|
||||||
|
};
|
||||||
|
self.set_storage_value(&key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn del(&self, key: String) -> Result<(), DBError> {
|
||||||
|
self.db.remove(&key).map_err(|e| DBError(e.to_string()))?;
|
||||||
|
self.types.remove(&key).map_err(|e| DBError(e.to_string()))?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exists(&self, key: &str) -> Result<bool, DBError> {
|
||||||
|
// Check with expiration
|
||||||
|
Ok(self.get_storage_value(key)?.is_some())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn keys(&self, pattern: &str) -> Result<Vec<String>, DBError> {
|
||||||
|
let mut keys = Vec::new();
|
||||||
|
for item in self.types.iter() {
|
||||||
|
let (key_bytes, _) = item.map_err(|e| DBError(e.to_string()))?;
|
||||||
|
let key = String::from_utf8_lossy(&key_bytes).to_string();
|
||||||
|
|
||||||
|
// Check if key is expired
|
||||||
|
if self.get_storage_value(&key)?.is_some() {
|
||||||
|
if Self::glob_match(pattern, &key) {
|
||||||
|
keys.push(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(keys)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn scan(&self, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError> {
|
||||||
|
let mut result = Vec::new();
|
||||||
|
let mut current_cursor = 0u64;
|
||||||
|
let limit = count.unwrap_or(10) as usize;
|
||||||
|
|
||||||
|
for item in self.types.iter() {
|
||||||
|
if current_cursor >= cursor {
|
||||||
|
let (key_bytes, type_bytes) = item.map_err(|e| DBError(e.to_string()))?;
|
||||||
|
let key = String::from_utf8_lossy(&key_bytes).to_string();
|
||||||
|
|
||||||
|
// Check pattern match
|
||||||
|
let matches = if let Some(pat) = pattern {
|
||||||
|
Self::glob_match(pat, &key)
|
||||||
|
} else {
|
||||||
|
true
|
||||||
|
};
|
||||||
|
|
||||||
|
if matches {
|
||||||
|
// Check if key is expired and get value
|
||||||
|
if let Some(storage_val) = self.get_storage_value(&key)? {
|
||||||
|
let value = match storage_val.value {
|
||||||
|
ValueType::String(s) => s,
|
||||||
|
_ => String::from_utf8_lossy(&type_bytes).to_string(),
|
||||||
|
};
|
||||||
|
result.push((key, value));
|
||||||
|
|
||||||
|
if result.len() >= limit {
|
||||||
|
current_cursor += 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
current_cursor += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
let next_cursor = if result.len() < limit { 0 } else { current_cursor };
|
||||||
|
Ok((next_cursor, result))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dbsize(&self) -> Result<i64, DBError> {
|
||||||
|
let mut count = 0i64;
|
||||||
|
for item in self.types.iter() {
|
||||||
|
let (key_bytes, _) = item.map_err(|e| DBError(e.to_string()))?;
|
||||||
|
let key = String::from_utf8_lossy(&key_bytes).to_string();
|
||||||
|
if self.get_storage_value(&key)?.is_some() {
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(count)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flushdb(&self) -> Result<(), DBError> {
|
||||||
|
self.db.clear().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
self.types.clear().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_key_type(&self, key: &str) -> Result<Option<String>, DBError> {
|
||||||
|
// First check if key exists (handles expiration)
|
||||||
|
if self.get_storage_value(key)?.is_some() {
|
||||||
|
match self.types.get(key).map_err(|e| DBError(e.to_string()))? {
|
||||||
|
Some(data) => Ok(Some(String::from_utf8_lossy(&data).to_string())),
|
||||||
|
None => Ok(None)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hash operations
|
||||||
|
fn hset(&self, key: &str, pairs: Vec<(String, String)>) -> Result<i64, DBError> {
|
||||||
|
let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue {
|
||||||
|
value: ValueType::Hash(HashMap::new()),
|
||||||
|
expires_at: None,
|
||||||
|
});
|
||||||
|
|
||||||
|
let hash = match &mut storage_val.value {
|
||||||
|
ValueType::Hash(h) => h,
|
||||||
|
_ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut new_fields = 0i64;
|
||||||
|
for (field, value) in pairs {
|
||||||
|
if !hash.contains_key(&field) {
|
||||||
|
new_fields += 1;
|
||||||
|
}
|
||||||
|
hash.insert(field, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(new_fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hget(&self, key: &str, field: &str) -> Result<Option<String>, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::Hash(h) => Ok(h.get(field).cloned()),
|
||||||
|
_ => Ok(None)
|
||||||
|
}
|
||||||
|
None => Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hgetall(&self, key: &str) -> Result<Vec<(String, String)>, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::Hash(h) => Ok(h.into_iter().collect()),
|
||||||
|
_ => Ok(Vec::new())
|
||||||
|
}
|
||||||
|
None => Ok(Vec::new())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hscan(&self, key: &str, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::Hash(h) => {
|
||||||
|
let mut result = Vec::new();
|
||||||
|
let mut current_cursor = 0u64;
|
||||||
|
let limit = count.unwrap_or(10) as usize;
|
||||||
|
|
||||||
|
for (field, value) in h.iter() {
|
||||||
|
if current_cursor >= cursor {
|
||||||
|
let matches = if let Some(pat) = pattern {
|
||||||
|
Self::glob_match(pat, field)
|
||||||
|
} else {
|
||||||
|
true
|
||||||
|
};
|
||||||
|
|
||||||
|
if matches {
|
||||||
|
result.push((field.clone(), value.clone()));
|
||||||
|
if result.len() >= limit {
|
||||||
|
current_cursor += 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
current_cursor += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
let next_cursor = if result.len() < limit { 0 } else { current_cursor };
|
||||||
|
Ok((next_cursor, result))
|
||||||
|
}
|
||||||
|
_ => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string()))
|
||||||
|
}
|
||||||
|
None => Ok((0, Vec::new()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hdel(&self, key: &str, fields: Vec<String>) -> Result<i64, DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(0)
|
||||||
|
};
|
||||||
|
|
||||||
|
let hash = match &mut storage_val.value {
|
||||||
|
ValueType::Hash(h) => h,
|
||||||
|
_ => return Ok(0)
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut deleted = 0i64;
|
||||||
|
for field in fields {
|
||||||
|
if hash.remove(&field).is_some() {
|
||||||
|
deleted += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if hash.is_empty() {
|
||||||
|
self.del(key.to_string())?;
|
||||||
|
} else {
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(deleted)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hexists(&self, key: &str, field: &str) -> Result<bool, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::Hash(h) => Ok(h.contains_key(field)),
|
||||||
|
_ => Ok(false)
|
||||||
|
}
|
||||||
|
None => Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hkeys(&self, key: &str) -> Result<Vec<String>, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::Hash(h) => Ok(h.keys().cloned().collect()),
|
||||||
|
_ => Ok(Vec::new())
|
||||||
|
}
|
||||||
|
None => Ok(Vec::new())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hvals(&self, key: &str) -> Result<Vec<String>, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::Hash(h) => Ok(h.values().cloned().collect()),
|
||||||
|
_ => Ok(Vec::new())
|
||||||
|
}
|
||||||
|
None => Ok(Vec::new())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hlen(&self, key: &str) -> Result<i64, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::Hash(h) => Ok(h.len() as i64),
|
||||||
|
_ => Ok(0)
|
||||||
|
}
|
||||||
|
None => Ok(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hmget(&self, key: &str, fields: Vec<String>) -> Result<Vec<Option<String>>, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::Hash(h) => {
|
||||||
|
Ok(fields.into_iter().map(|f| h.get(&f).cloned()).collect())
|
||||||
|
}
|
||||||
|
_ => Ok(fields.into_iter().map(|_| None).collect())
|
||||||
|
}
|
||||||
|
None => Ok(fields.into_iter().map(|_| None).collect())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hsetnx(&self, key: &str, field: &str, value: &str) -> Result<bool, DBError> {
|
||||||
|
let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue {
|
||||||
|
value: ValueType::Hash(HashMap::new()),
|
||||||
|
expires_at: None,
|
||||||
|
});
|
||||||
|
|
||||||
|
let hash = match &mut storage_val.value {
|
||||||
|
ValueType::Hash(h) => h,
|
||||||
|
_ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||||
|
};
|
||||||
|
|
||||||
|
if hash.contains_key(field) {
|
||||||
|
Ok(false)
|
||||||
|
} else {
|
||||||
|
hash.insert(field.to_string(), value.to_string());
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// List operations
|
||||||
|
fn lpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError> {
|
||||||
|
let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue {
|
||||||
|
value: ValueType::List(Vec::new()),
|
||||||
|
expires_at: None,
|
||||||
|
});
|
||||||
|
|
||||||
|
let list = match &mut storage_val.value {
|
||||||
|
ValueType::List(l) => l,
|
||||||
|
_ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||||
|
};
|
||||||
|
|
||||||
|
for element in elements.into_iter().rev() {
|
||||||
|
list.insert(0, element);
|
||||||
|
}
|
||||||
|
|
||||||
|
let len = list.len() as i64;
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(len)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError> {
|
||||||
|
let mut storage_val = self.get_storage_value(key)?.unwrap_or(StorageValue {
|
||||||
|
value: ValueType::List(Vec::new()),
|
||||||
|
expires_at: None,
|
||||||
|
});
|
||||||
|
|
||||||
|
let list = match &mut storage_val.value {
|
||||||
|
ValueType::List(l) => l,
|
||||||
|
_ => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||||
|
};
|
||||||
|
|
||||||
|
list.extend(elements);
|
||||||
|
let len = list.len() as i64;
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(len)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(Vec::new())
|
||||||
|
};
|
||||||
|
|
||||||
|
let list = match &mut storage_val.value {
|
||||||
|
ValueType::List(l) => l,
|
||||||
|
_ => return Ok(Vec::new())
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut result = Vec::new();
|
||||||
|
for _ in 0..count.min(list.len() as u64) {
|
||||||
|
if let Some(elem) = list.first() {
|
||||||
|
result.push(elem.clone());
|
||||||
|
list.remove(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if list.is_empty() {
|
||||||
|
self.del(key.to_string())?;
|
||||||
|
} else {
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(Vec::new())
|
||||||
|
};
|
||||||
|
|
||||||
|
let list = match &mut storage_val.value {
|
||||||
|
ValueType::List(l) => l,
|
||||||
|
_ => return Ok(Vec::new())
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut result = Vec::new();
|
||||||
|
for _ in 0..count.min(list.len() as u64) {
|
||||||
|
if let Some(elem) = list.pop() {
|
||||||
|
result.push(elem);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if list.is_empty() {
|
||||||
|
self.del(key.to_string())?;
|
||||||
|
} else {
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn llen(&self, key: &str) -> Result<i64, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::List(l) => Ok(l.len() as i64),
|
||||||
|
_ => Ok(0)
|
||||||
|
}
|
||||||
|
None => Ok(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lindex(&self, key: &str, index: i64) -> Result<Option<String>, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::List(list) => {
|
||||||
|
let actual_index = if index < 0 {
|
||||||
|
list.len() as i64 + index
|
||||||
|
} else {
|
||||||
|
index
|
||||||
|
};
|
||||||
|
|
||||||
|
if actual_index >= 0 && (actual_index as usize) < list.len() {
|
||||||
|
Ok(Some(list[actual_index as usize].clone()))
|
||||||
|
} else {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => Ok(None)
|
||||||
|
}
|
||||||
|
None => Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lrange(&self, key: &str, start: i64, stop: i64) -> Result<Vec<String>, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => match storage_val.value {
|
||||||
|
ValueType::List(list) => {
|
||||||
|
if list.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let len = list.len() as i64;
|
||||||
|
let start_idx = if start < 0 {
|
||||||
|
std::cmp::max(0, len + start)
|
||||||
|
} else {
|
||||||
|
std::cmp::min(start, len)
|
||||||
|
};
|
||||||
|
let stop_idx = if stop < 0 {
|
||||||
|
std::cmp::max(-1, len + stop)
|
||||||
|
} else {
|
||||||
|
std::cmp::min(stop, len - 1)
|
||||||
|
};
|
||||||
|
|
||||||
|
if start_idx > stop_idx || start_idx >= len {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let start_usize = start_idx as usize;
|
||||||
|
let stop_usize = (stop_idx + 1) as usize;
|
||||||
|
|
||||||
|
Ok(list[start_usize..std::cmp::min(stop_usize, list.len())].to_vec())
|
||||||
|
}
|
||||||
|
_ => Ok(Vec::new())
|
||||||
|
}
|
||||||
|
None => Ok(Vec::new())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ltrim(&self, key: &str, start: i64, stop: i64) -> Result<(), DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(())
|
||||||
|
};
|
||||||
|
|
||||||
|
let list = match &mut storage_val.value {
|
||||||
|
ValueType::List(l) => l,
|
||||||
|
_ => return Ok(())
|
||||||
|
};
|
||||||
|
|
||||||
|
if list.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let len = list.len() as i64;
|
||||||
|
let start_idx = if start < 0 {
|
||||||
|
std::cmp::max(0, len + start)
|
||||||
|
} else {
|
||||||
|
std::cmp::min(start, len)
|
||||||
|
};
|
||||||
|
let stop_idx = if stop < 0 {
|
||||||
|
std::cmp::max(-1, len + stop)
|
||||||
|
} else {
|
||||||
|
std::cmp::min(stop, len - 1)
|
||||||
|
};
|
||||||
|
|
||||||
|
if start_idx > stop_idx || start_idx >= len {
|
||||||
|
self.del(key.to_string())?;
|
||||||
|
} else {
|
||||||
|
let start_usize = start_idx as usize;
|
||||||
|
let stop_usize = (stop_idx + 1) as usize;
|
||||||
|
*list = list[start_usize..std::cmp::min(stop_usize, list.len())].to_vec();
|
||||||
|
|
||||||
|
if list.is_empty() {
|
||||||
|
self.del(key.to_string())?;
|
||||||
|
} else {
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lrem(&self, key: &str, count: i64, element: &str) -> Result<i64, DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(0)
|
||||||
|
};
|
||||||
|
|
||||||
|
let list = match &mut storage_val.value {
|
||||||
|
ValueType::List(l) => l,
|
||||||
|
_ => return Ok(0)
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut removed = 0i64;
|
||||||
|
|
||||||
|
if count == 0 {
|
||||||
|
// Remove all occurrences
|
||||||
|
let original_len = list.len();
|
||||||
|
list.retain(|x| x != element);
|
||||||
|
removed = (original_len - list.len()) as i64;
|
||||||
|
} else if count > 0 {
|
||||||
|
// Remove first count occurrences
|
||||||
|
let mut to_remove = count as usize;
|
||||||
|
list.retain(|x| {
|
||||||
|
if x == element && to_remove > 0 {
|
||||||
|
to_remove -= 1;
|
||||||
|
removed += 1;
|
||||||
|
false
|
||||||
|
} else {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// Remove last |count| occurrences
|
||||||
|
let mut to_remove = (-count) as usize;
|
||||||
|
for i in (0..list.len()).rev() {
|
||||||
|
if list[i] == element && to_remove > 0 {
|
||||||
|
list.remove(i);
|
||||||
|
to_remove -= 1;
|
||||||
|
removed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if list.is_empty() {
|
||||||
|
self.del(key.to_string())?;
|
||||||
|
} else {
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(removed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expiration
|
||||||
|
fn ttl(&self, key: &str) -> Result<i64, DBError> {
|
||||||
|
match self.get_storage_value(key)? {
|
||||||
|
Some(storage_val) => {
|
||||||
|
if let Some(expires_at) = storage_val.expires_at {
|
||||||
|
let now = Self::now_millis();
|
||||||
|
if now >= expires_at {
|
||||||
|
Ok(-2) // Key has expired
|
||||||
|
} else {
|
||||||
|
Ok(((expires_at - now) / 1000) as i64) // TTL in seconds
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Ok(-1) // Key exists but has no expiration
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => Ok(-2) // Key does not exist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expire_seconds(&self, key: &str, secs: u64) -> Result<bool, DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(false)
|
||||||
|
};
|
||||||
|
|
||||||
|
storage_val.expires_at = Some(Self::now_millis() + (secs as u128) * 1000);
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pexpire_millis(&self, key: &str, ms: u128) -> Result<bool, DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(false)
|
||||||
|
};
|
||||||
|
|
||||||
|
storage_val.expires_at = Some(Self::now_millis() + ms);
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn persist(&self, key: &str) -> Result<bool, DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(false)
|
||||||
|
};
|
||||||
|
|
||||||
|
if storage_val.expires_at.is_some() {
|
||||||
|
storage_val.expires_at = None;
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(true)
|
||||||
|
} else {
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expire_at_seconds(&self, key: &str, ts_secs: i64) -> Result<bool, DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(false)
|
||||||
|
};
|
||||||
|
|
||||||
|
let expires_at_ms: u128 = if ts_secs <= 0 { 0 } else { (ts_secs as u128) * 1000 };
|
||||||
|
storage_val.expires_at = Some(expires_at_ms);
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pexpire_at_millis(&self, key: &str, ts_ms: i64) -> Result<bool, DBError> {
|
||||||
|
let mut storage_val = match self.get_storage_value(key)? {
|
||||||
|
Some(sv) => sv,
|
||||||
|
None => return Ok(false)
|
||||||
|
};
|
||||||
|
|
||||||
|
let expires_at_ms: u128 = if ts_ms <= 0 { 0 } else { ts_ms as u128 };
|
||||||
|
storage_val.expires_at = Some(expires_at_ms);
|
||||||
|
self.set_storage_value(key, storage_val)?;
|
||||||
|
self.db.flush().map_err(|e| DBError(e.to_string()))?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_encrypted(&self) -> bool {
|
||||||
|
self.crypto.is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clone_arc(&self) -> Arc<dyn StorageBackend> {
|
||||||
|
// Note: This is a simplified clone - in production you might want to
|
||||||
|
// handle this differently as sled::Db is already Arc internally
|
||||||
|
Arc::new(SledStorage {
|
||||||
|
db: self.db.clone(),
|
||||||
|
types: self.types.clone(),
|
||||||
|
crypto: self.crypto.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
57
herodb/src/storage_trait.rs
Normal file
57
herodb/src/storage_trait.rs
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
// src/storage_trait.rs
|
||||||
|
use crate::error::DBError;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
pub trait StorageBackend: Send + Sync {
|
||||||
|
// Basic key operations
|
||||||
|
fn get(&self, key: &str) -> Result<Option<String>, DBError>;
|
||||||
|
fn set(&self, key: String, value: String) -> Result<(), DBError>;
|
||||||
|
fn setx(&self, key: String, value: String, expire_ms: u128) -> Result<(), DBError>;
|
||||||
|
fn del(&self, key: String) -> Result<(), DBError>;
|
||||||
|
fn exists(&self, key: &str) -> Result<bool, DBError>;
|
||||||
|
fn keys(&self, pattern: &str) -> Result<Vec<String>, DBError>;
|
||||||
|
fn dbsize(&self) -> Result<i64, DBError>;
|
||||||
|
fn flushdb(&self) -> Result<(), DBError>;
|
||||||
|
fn get_key_type(&self, key: &str) -> Result<Option<String>, DBError>;
|
||||||
|
|
||||||
|
// Scanning
|
||||||
|
fn scan(&self, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError>;
|
||||||
|
fn hscan(&self, key: &str, cursor: u64, pattern: Option<&str>, count: Option<u64>) -> Result<(u64, Vec<(String, String)>), DBError>;
|
||||||
|
|
||||||
|
// Hash operations
|
||||||
|
fn hset(&self, key: &str, pairs: Vec<(String, String)>) -> Result<i64, DBError>;
|
||||||
|
fn hget(&self, key: &str, field: &str) -> Result<Option<String>, DBError>;
|
||||||
|
fn hgetall(&self, key: &str) -> Result<Vec<(String, String)>, DBError>;
|
||||||
|
fn hdel(&self, key: &str, fields: Vec<String>) -> Result<i64, DBError>;
|
||||||
|
fn hexists(&self, key: &str, field: &str) -> Result<bool, DBError>;
|
||||||
|
fn hkeys(&self, key: &str) -> Result<Vec<String>, DBError>;
|
||||||
|
fn hvals(&self, key: &str) -> Result<Vec<String>, DBError>;
|
||||||
|
fn hlen(&self, key: &str) -> Result<i64, DBError>;
|
||||||
|
fn hmget(&self, key: &str, fields: Vec<String>) -> Result<Vec<Option<String>>, DBError>;
|
||||||
|
fn hsetnx(&self, key: &str, field: &str, value: &str) -> Result<bool, DBError>;
|
||||||
|
|
||||||
|
// List operations
|
||||||
|
fn lpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError>;
|
||||||
|
fn rpush(&self, key: &str, elements: Vec<String>) -> Result<i64, DBError>;
|
||||||
|
fn lpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError>;
|
||||||
|
fn rpop(&self, key: &str, count: u64) -> Result<Vec<String>, DBError>;
|
||||||
|
fn llen(&self, key: &str) -> Result<i64, DBError>;
|
||||||
|
fn lindex(&self, key: &str, index: i64) -> Result<Option<String>, DBError>;
|
||||||
|
fn lrange(&self, key: &str, start: i64, stop: i64) -> Result<Vec<String>, DBError>;
|
||||||
|
fn ltrim(&self, key: &str, start: i64, stop: i64) -> Result<(), DBError>;
|
||||||
|
fn lrem(&self, key: &str, count: i64, element: &str) -> Result<i64, DBError>;
|
||||||
|
|
||||||
|
// Expiration
|
||||||
|
fn ttl(&self, key: &str) -> Result<i64, DBError>;
|
||||||
|
fn expire_seconds(&self, key: &str, secs: u64) -> Result<bool, DBError>;
|
||||||
|
fn pexpire_millis(&self, key: &str, ms: u128) -> Result<bool, DBError>;
|
||||||
|
fn persist(&self, key: &str) -> Result<bool, DBError>;
|
||||||
|
fn expire_at_seconds(&self, key: &str, ts_secs: i64) -> Result<bool, DBError>;
|
||||||
|
fn pexpire_at_millis(&self, key: &str, ts_ms: i64) -> Result<bool, DBError>;
|
||||||
|
|
||||||
|
// Metadata
|
||||||
|
fn is_encrypted(&self) -> bool;
|
||||||
|
|
||||||
|
// Clone to Arc for sharing
|
||||||
|
fn clone_arc(&self) -> Arc<dyn StorageBackend>;
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user