...

Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "herodb"
 version = "0.0.1"
-authors = ["ThreeFold Tech NV"]
-edition = "2024"
+authors = ["Pin Fang <fpfangpin@hotmail.com>"]
+edition = "2021"
 
 [dependencies]
 anyhow = "1.0.59"
@@ -23,18 +23,19 @@ sha2 = "0.10"
 age = "0.10"
 secrecy = "0.8"
 ed25519-dalek = "2"
-x25519-dalek = "2"
 base64 = "0.22"
-jsonrpsee = { version = "0.26.0", features = ["http-client", "ws-client", "server", "macros"] }
-tantivy = "0.25.0"
-arrow-schema = "55.2.0"
-arrow-array = "55.2.0"
-lance = "0.37.0"
-lance-index = "0.37.0"
-arrow = "55.2.0"
-lancedb = "0.22.1"
-uuid = "1.18.1"
-ureq = { version = "2.10.0", features = ["json", "tls"] }
+# Lance vector database dependencies
+lance = "0.33"
+lance-index = "0.33"
+lance-linalg = "0.33"
+# Use Arrow version compatible with Lance 0.33
+arrow = "55.2"
+arrow-array = "55.2"
+arrow-schema = "55.2"
+parquet = "55.2"
+uuid = { version = "1.10", features = ["v4"] }
+reqwest = { version = "0.11", features = ["json"] }
+image = "0.25"
 
 [dev-dependencies]
 redis = { version = "0.24", features = ["aio", "tokio-comp"] }

README.md

@@ -17,8 +17,6 @@ The main purpose of HeroDB is to offer a lightweight, embeddable, and Redis-comp
 - **Expiration**: Time-to-live (TTL) functionality for keys.
 - **Scanning**: Cursor-based iteration for keys and hash fields (`SCAN`, `HSCAN`).
 - **AGE Cryptography Commands**: HeroDB-specific extensions for cryptographic operations.
-- **Symmetric Encryption**: Stateless symmetric encryption using XChaCha20-Poly1305.
-- **Admin Database 0**: Centralized control for database management, access control, and per-database encryption.
 
 ## Quick Start
 
@@ -32,14 +30,31 @@ cargo build --release
 
 ### Running HeroDB
 
-Launch HeroDB with the required `--admin-secret` flag, which encrypts the admin database (DB 0) and authorizes admin access. Optional flags include `--dir` for the database directory, `--port` for the TCP port (default 6379), `--sled` for the sled backend, and `--enable-rpc` to start the JSON-RPC management server on port 8080.
+You can start HeroDB with different backends and encryption options:
+
+#### Default `redb` Backend
 
-Example:
 ```bash
-./target/release/herodb --dir /tmp/herodb --admin-secret myadminsecret --port 6379 --enable-rpc
+./target/release/herodb --dir /tmp/herodb_redb --port 6379
 ```
 
-For detailed launch options, see [Basics](docs/basics.md).
+#### `sled` Backend
+
+```bash
+./target/release/herodb --dir /tmp/herodb_sled --port 6379 --sled
+```
+
+#### `redb` with Encryption
+
+```bash
+./target/release/herodb --dir /tmp/herodb_encrypted --port 6379 --encrypt --key mysecretkey
+```
+
+#### `sled` with Encryption
+
+```bash
+./target/release/herodb --dir /tmp/herodb_sled_encrypted --port 6379 --sled --encrypt --key mysecretkey
+```
 
 ## Usage with Redis Clients
 
@@ -47,44 +62,24 @@ HeroDB can be interacted with using any standard Redis client, such as `redis-cl
 
 ### Example with `redis-cli`
 
-Connections start with no database selected. You must SELECT a database first.
-
-- To work in the admin database (DB 0), authenticate with the admin secret:
 ```bash
-redis-cli -p 6379 SELECT 0 KEY myadminsecret
 redis-cli -p 6379 SET mykey "Hello from HeroDB!"
 redis-cli -p 6379 GET mykey
 # → "Hello from HeroDB!"
-```
 
-- To use a user database, first create one via the JSON-RPC API (see docs/rpc_examples.md), then select it:
-```bash
-# Suppose RPC created database id 1
-redis-cli -p 6379 SELECT 1
 redis-cli -p 6379 HSET user:1 name "Alice" age "30"
 redis-cli -p 6379 HGET user:1 name
 # → "Alice"
+
 redis-cli -p 6379 SCAN 0 MATCH user:* COUNT 10
+# → 1) "0"
+#    2) 1) "user:1"
 ```
 
-## Cryptography
-
-HeroDB supports asymmetric encryption/signatures via AGE commands (X25519 for encryption, Ed25519 for signatures) in stateless or key-managed modes, and symmetric encryption via SYM commands. Keys are persisted in the admin database (DB 0) for managed modes.
-
-For details, see [AGE Cryptography](docs/age.md) and [Basics](docs/basics.md).
-
-## Database Management
-
-Databases are managed via JSON-RPC API, with metadata stored in the encrypted admin database (DB 0). Databases are public by default upon creation; use RPC to set them private, requiring access keys for SELECT operations (read or readwrite based on permissions). This includes per-database encryption keys, access control, and lifecycle management.
-
-For examples, see [JSON-RPC Examples](docs/rpc_examples.md) and [Admin DB 0 Model](docs/admin.md).
-
 ## Documentation
 
 For more detailed information on commands, features, and advanced usage, please refer to the documentation:
 
 - [Basics](docs/basics.md)
 - [Supported Commands](docs/cmds.md)
-- [AGE Cryptography](docs/age.md)
-- [Admin DB 0 Model (access control, per-db encryption)](docs/admin.md)
-- [JSON-RPC Examples (management API)](docs/rpc_examples.md)
+- [AGE Cryptography](docs/age.md)

docs/admin.md

@@ -1,182 +0,0 @@
-# Admin Database 0 (`0.db`)
-
-This page explains what the Admin Database `DB 0` is, why HeroDB uses it, and how to work with it as a developer and end-user. It’s a practical guide covering how databases are created, listed, secured with access keys, and encrypted using per-database secrets.
-
-## What is `DB 0`?
-
-`DB 0` is the control-plane for a HeroDB instance. It stores metadata for all user databases (`db_id >= 1`) so the server can:
-- Know which databases exist (without scanning the filesystem)
-- Enforce access control (public/private with access keys)
-- Enforce per-database encryption (whether a given database must be opened encrypted and with which write-only key)
-
-`DB 0` itself is always encrypted with the admin secret (the process-level secret provided at startup).
-
-## How `DB 0` is created and secured
-
-- `DB 0` lives at `<base_dir>/0.db`
-- It is always encrypted using the `admin secret` provided at process startup (using the `--admin-secret <secret>` CLI flag)
-- Only clients that provide the correct admin secret can `SELECT 0` (see “`SELECT` + `KEY`” below)
-
-At startup, the server bootstraps `DB 0` (initializes counters and structures) if it’s missing.
-
-## Metadata stored in `DB 0`
-
-Keys in `DB 0` (internal layout, but useful to understand how things work):
-
-- `admin:next_id`
-  - String counter holding the next id to allocate (initialized to `"1"`)
-
-- `admin:dbs`
-  - A hash acting as a set of existing database ids
-  - field = id (as string), value = `"1"`
-
-- `meta:db:<id>`
-  - A hash holding db-level metadata
-  - field `public` = `"true"` or `"false"` (defaults to `true` if missing)
-
-- `meta:db:<id>:keys`
-  - A hash mapping access-key hashes to the string `Permission:created_at_seconds`
-  - Examples: `Read:1713456789` or `ReadWrite:1713456789`
-  - The plaintext access keys are never stored; only their `SHA-256` hashes are kept
-
-- `meta:db:<id>:enc`
-   - A string holding the per-database encryption key used to open `<id>.db` encrypted
-   - This value is write-only from the perspective of the management APIs (it’s set at creation and never returned)
-
-- `age:key:<name>`
-   - Base64-encoded X25519 recipient (public encryption key) for named AGE keys
-- `age:privkey:<name>`
-   - Base64-encoded X25519 identity (secret encryption key) for named AGE keys
-- `age:signpub:<name>`
-   - Base64-encoded Ed25519 verify public key for named AGE keys
-- `age:signpriv:<name>`
-   - Base64-encoded Ed25519 signing secret key for named AGE keys
-
-> You don’t need to manipulate these keys directly; they’re listed to clarify the model. AGE keys are managed via AGE commands.
-
-## Database lifecycle
-
-1) Create a database (via JSON-RPC)
-- The server allocates an id from `admin:next_id`, registers it in `admin:dbs`, and defaults the database to `public=true`
-- If you pass an optional `encryption_key` during creation, the server persists it in `meta:db:<id>:enc`. That database will be opened in encrypted mode from then on
-
-2) Open and use a database
-- Clients select a database over RESP using `SELECT`
-- Authorization and encryption state are enforced using `DB 0` metadata
-
-3) Delete database files
-- Removing `<id>.db` removes the physical storage
-- `DB 0` remains the source of truth for existence and may be updated by future management methods as the system evolves
-
-## Access control model
-
-- Public database (default)
-  - Anyone can `SELECT <id>` with no key, and will get `ReadWrite` permission
-- Private database
-  - You must provide an access key when selecting the database
-  - The server hashes the provided key with `SHA-256` and checks membership in `meta:db:<id>:keys`
-  - Permissions are `Read` or `ReadWrite` depending on how the key was added
-- Admin `DB 0`
-  - Requires the exact admin secret as the `KEY` argument to `SELECT 0`
-  - Permission is `ReadWrite` when the secret matches
-
-Connections start with no database selected. Any command that requires storage (GET, SET, H*, L*, SCAN, etc.) will return an error until you issue a SELECT to choose a database. Admin DB 0 is never accessible without authenticating via SELECT 0 KEY &lt;admin_secret&gt;.
-### How to select databases with optional `KEY`
-
-- Public DB (no key required)
-  - `SELECT <id>`
-
-- Private DB (access key required)
-  - `SELECT <id> KEY <plaintext_key>`
-
-- Admin `DB 0` (admin secret required)
-  - `SELECT 0 KEY <admin_secret>`
-
-Examples (using `redis-cli`):
-```bash
-# Public database
-redis-cli -p $PORT SELECT 1
-# → OK
-
-# Private database
-redis-cli -p $PORT SELECT 2 KEY my-db2-access-key
-# → OK
-
-# Admin DB 0
-redis-cli -p $PORT SELECT 0 KEY my-admin-secret
-# → OK
-```
-
-## Per-database encryption
-
-- At database creation, you can provide an optional per-db encryption key
-- If provided, the server persists that key in `DB 0` as `meta:db:<id>:enc`
-- When you later open the database, the engine checks whether `meta:db:<id>:enc` exists to decide if it must open `<id>.db` in encrypted mode
-- The per-db key is not returned by RPC—it is considered write-only configuration data
-
-Operationally:
-- Create with encryption: pass a non-null `encryption_key` to the `createDatabase` RPC
-- Open later: simply `SELECT` the database; encryption is transparent to clients
-
-## Management via JSON-RPC
-
-You can manage databases using the management RPC (namespaced `herodb.*`). Typical operations:
-- `createDatabase(backend, config, encryption_key?)`
-  - Allocates a new id, sets optional encryption key
-- `listDatabases()`
-  - Lists database ids and info (including whether storage is currently encrypted)
-- `getDatabaseInfo(db_id)`
-  - Returns details: backend, encrypted flag, size on disk, `key_count`, timestamps, etc.
-- `addAccessKey(db_id, key, permissions)`
-  - Adds a `Read` or `ReadWrite` access key (permissions = `"read"` | `"readwrite"`)
-- `listAccessKeys(db_id)`
-  - Returns hashes and permissions; you can use these hashes to delete keys
-- `deleteAccessKey(db_id, key_hash)`
-  - Removes a key by its hash
-- `setDatabasePublic(db_id, public)`
-  - Toggles public/private
-
-Copyable JSON examples are provided in the [RPC examples documentation](./rpc_examples.md).
-
-## Typical flows
-
-1) Public, unencrypted database
-- Create a new database without an encryption key
-- Clients can immediately `SELECT <id>` without a key
-- You can later make it private and add keys if needed
-
-2) Private, encrypted database
-- Create passing an `encryption_key`
-- Mark it private (`setDatabasePublic false`) and add access keys
-- Clients must use `SELECT <id> KEY <plaintext_access_key>`
-- Storage opens in encrypted mode automatically
-
-## Security notes
-
-- Only `SHA-256` hashes of access keys are stored in `DB 0`; keep plaintext keys safe on the client side
-- The per-db encryption key is never exposed via the API after it is set
-- The admin secret must be kept secure; anyone with it can `SELECT 0` and perform administrative actions
-
-## Troubleshooting
-
-- `ERR invalid access key` when selecting a private db
-  - Ensure you passed the `KEY` argument: `SELECT <id> KEY <plaintext_key>`
-  - If you recently added the key, confirm the permissions and that you used the exact plaintext (hash must match)
-
-- `Database X not found`
-  - The id isn’t registered in `DB 0` (`admin:dbs`). Use the management APIs to create or list databases
-
-- Cannot `SELECT 0`
-  - The `KEY` must be the exact admin secret passed at server startup
-
-## Reference
-
-- Admin metadata lives in `DB 0` (`0.db`) and controls:
-  - Existence: `admin:dbs`
-  - Access: `meta:db:<id>.public` and `meta:db:<id>:keys`
-  - Encryption: `meta:db:<id>:enc`
-
-For command examples and management payloads:
-- RESP command basics: `docs/basics.md`
-- Supported commands: `docs/cmds.md`
-- JSON-RPC examples: `docs/rpc_examples.md`

docs/age.md

@@ -1,96 +1,188 @@
-# HeroDB AGE Cryptography
+# HeroDB AGE usage: Stateless vs Key‑Managed
 
-HeroDB provides AGE-based asymmetric encryption and digital signatures over the Redis protocol using X25519 for encryption and Ed25519 for signatures. Keys can be used in stateless (ephemeral) or key-managed (persistent, named) modes.
+This document explains how to use the AGE cryptography commands exposed by HeroDB over the Redis protocol in two modes:
+- Stateless (ephemeral keys; nothing stored on the server)
+- Key‑managed (server‑persisted, named keys)
 
-In key-managed mode, HeroDB uses a unified keypair concept: a single Ed25519 signing key is deterministically derived into X25519 keys for encryption, allowing one keypair to handle both encryption and signatures transparently.
+If you are new to the codebase, the exact tests that exercise these behaviors are:
+- [rust.test_07_age_stateless_suite()](herodb/tests/usage_suite.rs:495)
+- [rust.test_08_age_persistent_named_suite()](herodb/tests/usage_suite.rs:555)
 
-## Cryptographic Algorithms
+Implementation entry points:
+- [herodb/src/age.rs](herodb/src/age.rs)
+- Dispatch from [herodb/src/cmd.rs](herodb/src/cmd.rs)
 
-### X25519 (Encryption)
-- Elliptic-curve Diffie-Hellman key exchange for symmetric key derivation.
-- Used for encrypting/decrypting messages.
+Note: Database-at-rest encryption flags in the test harness are unrelated to AGE commands; those flags control storage-level encryption of DB files. See the harness near [rust.start_test_server()](herodb/tests/usage_suite.rs:10).
 
-### Ed25519 (Signatures)
-- EdDSA digital signatures for message authentication.
-- Used for signing/verifying messages.
+## Quick start
 
-### Key Derivation
-Ed25519 signing keys are deterministically converted to X25519 keys for encryption. This enables a single keypair to support both operations without additional keys. Derivation uses the Ed25519 secret scalar clamped for X25519.
-
-In named keypairs, Ed25519 keys are stored, and X25519 keys are derived on-demand and cached.
-
-## Stateless Mode (Ephemeral Keys)
-No server-side storage; keys are provided with each command.
-
-Available commands:
-- `AGE GENENC`: Generate ephemeral X25519 keypair. Returns `[recipient, identity]`.
-- `AGE GENSIGN`: Generate ephemeral Ed25519 keypair. Returns `[verify_pub, sign_secret]`.
-- `AGE ENCRYPT <recipient> <message>`: Encrypt message. Returns base64 ciphertext.
-- `AGE DECRYPT <identity> <ciphertext_b64>`: Decrypt ciphertext. Returns plaintext.
-- `AGE SIGN <sign_secret> <message>`: Sign message. Returns base64 signature.
-- `AGE VERIFY <verify_pub> <message> <signature_b64>`: Verify signature. Returns 1 (valid) or 0 (invalid).
-
-Example:
+Assuming the server is running on localhost on some $PORT:
 ```bash
-redis-cli AGE GENENC
-# → 1) "age1qz..."  # recipient (X25519 public)
-#    2) "AGE-SECRET-KEY-1..."  # identity (X25519 secret)
+~/code/git.ourworld.tf/herocode/herodb/herodb/build.sh
+~/code/git.ourworld.tf/herocode/herodb/target/release/herodb --dir /tmp/data --debug --$PORT 6381 --encryption-key 1234 --encrypt
+```
 
-redis-cli AGE ENCRYPT "age1qz..." "hello"
-# → base64_ciphertext
 
-redis-cli AGE DECRYPT "AGE-SECRET-KEY-1..." base64_ciphertext
+```bash
+export PORT=6381
+# Generate an ephemeral keypair and encrypt/decrypt a message (stateless mode)
+redis-cli -p $PORT AGE GENENC
+# → returns an array: [recipient, identity]
+
+redis-cli -p $PORT AGE ENCRYPT <recipient> "hello world"
+# → returns ciphertext (base64 in a bulk string)
+
+redis-cli -p $PORT AGE DECRYPT <identity> <ciphertext_b64>
+# → returns "hello world"
+```
+
+For key‑managed mode, generate a named key once and reference it by name afterwards:
+
+```bash
+redis-cli -p $PORT AGE KEYGEN app1
+# → persists encryption keypair under name "app1"
+
+redis-cli -p $PORT AGE ENCRYPTNAME app1 "hello"
+redis-cli -p $PORT AGE DECRYPTNAME app1 <ciphertext_b64>
+```
+
+## Stateless AGE (ephemeral)
+
+Characteristics
+
+- No server‑side storage of keys.
+- You pass the actual key material with every call.
+- Not listable via AGE LIST.
+
+Commands and examples
+
+1) Ephemeral encryption keys
+
+```bash
+# Generate an ephemeral encryption keypair 
+redis-cli -p $PORT AGE GENENC
+# Example output (abridged):
+# 1) "age1qz..."          # recipient (public key) = can be used by others e.g. to verify what I sign
+# 2) "AGE-SECRET-KEY-1..." # identity (secret) = is like my private, cannot lose this one
+
+# Encrypt with the recipient public key
+redis-cli -p $PORT AGE ENCRYPT "age1qz..." "hello world"
+
+# → returns bulk string payload: base64 ciphertext (encrypted content)
+
+# Decrypt with the identity (secret) in other words your private key
+redis-cli -p $PORT AGE DECRYPT "AGE-SECRET-KEY-1..." "<ciphertext_b64>"
+# → "hello world"
+```
+
+2) Ephemeral signing keys
+
+> ? is this same as my private key
+
+```bash
+
+# Generate an ephemeral signing keypair
+redis-cli -p $PORT AGE GENSIGN
+# Example output:
+# 1) "<verify_pub_b64>"
+# 2) "<sign_secret_b64>"
+
+# Sign a message with the secret
+redis-cli -p $PORT AGE SIGN "<sign_secret_b64>" "msg"
+# → returns "<signature_b64>"
+
+# Verify with the public key
+redis-cli -p $PORT AGE VERIFY "<verify_pub_b64>" "msg" "<signature_b64>"
+# → 1 (valid) or 0 (invalid)
+```
+
+When to use
+- You do not want the server to store private keys.
+- You already manage key material on the client side.
+- You need ad‑hoc operations without persistence.
+
+Reference test: [rust.test_07_age_stateless_suite()](herodb/tests/usage_suite.rs:495)
+
+## Key‑managed AGE (persistent, named)
+
+Characteristics
+- Server generates and persists keypairs under a chosen name.
+- Clients refer to keys by name; raw secrets are not supplied on each call.
+- Keys are discoverable via AGE LIST.
+
+Commands and examples
+
+1) Named encryption keys
+
+```bash
+# Create/persist a named encryption keypair
+redis-cli -p $PORT AGE KEYGEN app1
+# → returns [recipient, identity] but also stores them under name "app1"
+
+> TODO: should not return identity (security, but there can be separate function to export it e.g. AGE EXPORTKEY app1)
+
+# Encrypt using the stored public key
+redis-cli -p $PORT AGE ENCRYPTNAME app1 "hello"
+# → returns bulk string payload: base64 ciphertext
+
+# Decrypt using the stored secret
+redis-cli -p $PORT AGE DECRYPTNAME app1 "<ciphertext_b64>"
 # → "hello"
 ```
 
-## Key-Managed Mode (Persistent Named Keys)
-Keys are stored server-side under names. Supports unified keypairs for both encryption and signatures.
+2) Named signing keys
 
-Available commands:
-- `AGE KEYGEN <name>`: Generate and store unified keypair. Returns `[recipient, identity]` in age format.
-- `AGE SIGNKEYGEN <name>`: Generate and store Ed25519 signing keypair. Returns `[verify_pub, sign_secret]`.
-- `AGE ENCRYPTNAME <name> <message>`: Encrypt with named key. Returns base64 ciphertext.
-- `AGE DECRYPTNAME <name> <ciphertext_b64>`: Decrypt with named key. Returns plaintext.
-- `AGE SIGNNAME <name> <message>`: Sign with named key. Returns base64 signature.
-- `AGE VERIFYNAME <name> <message> <signature_b64>`: Verify with named key. Returns 1 or 0.
-- `AGE LIST`: List all stored key names. Returns sorted array of names.
-
-### AGE LIST Output
-Returns a flat, deduplicated, sorted array of key names (strings). Each name corresponds to a stored keypair, which may include encryption keys (X25519), signing keys (Ed25519), or both.
-
-Output format: `["name1", "name2", ...]`
-
-Example:
 ```bash
-redis-cli AGE LIST
-# →  1) "<named_keypair_1>"
-#    2) "<named_keypair_2>"
+# Create/persist a named signing keypair
+redis-cli -p $PORT AGE SIGNKEYGEN app1
+# → returns [verify_pub_b64, sign_secret_b64] and stores under name "app1"
+
+> TODO: should not return sign_secret_b64 (for security, but there can be separate function to export it e.g. AGE EXPORTSIGNKEY app1)
+
+# Sign using the stored secret
+redis-cli -p $PORT AGE SIGNNAME app1 "msg"
+# → returns "<signature_b64>"
+
+# Verify using the stored public key
+redis-cli -p $PORT AGE VERIFYNAME app1 "msg" "<signature_b64>"
+# → 1 (valid) or 0 (invalid)
 ```
 
-For unified keypairs (from `AGE KEYGEN`), the name handles both encryption (derived X25519) and signatures (stored Ed25519) transparently.
+3) List stored AGE keys
 
-Example with named keys:
 ```bash
-redis-cli AGE KEYGEN app1
-# →  1) "age1..."  # recipient
-#    2) "AGE-SECRET-KEY-1..."  # identity
-
-redis-cli AGE ENCRYPTNAME app1 "secret message"
-# → base64_ciphertext
-
-redis-cli AGE DECRYPTNAME app1 base64_ciphertext
-# → "secret message"
-
-redis-cli AGE SIGNNAME app1 "message"
-# → base64_signature
-
-redis-cli AGE VERIFYNAME app1 "message" base64_signature
-# → 1
+redis-cli -p $PORT AGE LIST
+# Example output includes labels such as "encpub" and your key names (e.g., "app1")
 ```
 
-## Choosing a Mode
-- **Stateless**: For ad-hoc operations without persistence; client manages keys.
-- **Key-managed**: For centralized key lifecycle; server stores keys for convenience and discoverability.
+When to use
+- You want centralized key storage/rotation and fewer secrets on the client.
+- You need names/labels for workflows and can trust the server with secrets.
+- You want discoverability (AGE LIST) and simpler client commands.
 
-Implementation: [herodb/src/age.rs](herodb/src/age.rs) <br> 
-Tests: [herodb/tests/usage_suite.rs](herodb/tests/usage_suite.rs)
+Reference test: [rust.test_08_age_persistent_named_suite()](herodb/tests/usage_suite.rs:555)
+
+## Choosing a mode
+
+- Prefer Stateless when:
+  - Minimizing server trust for secret material is the priority.
+  - Clients already have a secure mechanism to store/distribute keys.
+- Prefer Key‑managed when:
+  - Centralized lifecycle, naming, and discoverability are beneficial.
+  - You plan to integrate rotation, ACLs, or auditability on the server side.
+
+## Security notes
+
+- Treat identities and signing secrets as sensitive; avoid logging them.
+- For key‑managed mode, ensure server storage (and backups) are protected.
+- AGE operations here are application‑level crypto and are distinct from database-at-rest encryption configured in the test harness.
+
+## Repository pointers
+
+- Stateless examples in tests: [rust.test_07_age_stateless_suite()](herodb/tests/usage_suite.rs:495)
+- Key‑managed examples in tests: [rust.test_08_age_persistent_named_suite()](herodb/tests/usage_suite.rs:555)
+- AGE implementation: [herodb/src/age.rs](herodb/src/age.rs)
+- Command dispatch: [herodb/src/cmd.rs](herodb/src/cmd.rs)
+- Bash demo: [herodb/examples/age_bash_demo.sh](herodb/examples/age_bash_demo.sh)
+- Rust persistent demo: [herodb/examples/age_persist_demo.rs](herodb/examples/age_persist_demo.rs)
+- Additional notes: [herodb/instructions/encrypt.md](herodb/instructions/encrypt.md)

docs/basics.md

@@ -1,58 +1,4 @@
-# HeroDB Basics
-
-## Launching HeroDB
-
-To launch HeroDB, use the binary with required and optional flags. The `--admin-secret` flag is mandatory, encrypting the admin database (DB 0) and authorizing admin access.
-
-### Launch Flags
-- `--dir <path>`: Directory for database files (default: current directory).
-- `--port <port>`: TCP port for Redis protocol (default: 6379).
-- `--debug`: Enable debug logging.
-- `--sled`: Use Sled backend (default: Redb).
-- `--enable-rpc`: Start JSON-RPC management server on port 8080.
-- `--rpc-port <port>`: Custom RPC port (default: 8080).
-- `--admin-secret <secret>`: Required secret for DB 0 encryption and admin access.
-
-Example:
-```bash
-./target/release/herodb --dir /tmp/herodb --admin-secret mysecret --port 6379 --enable-rpc
-```
-
-Deprecated flags (`--encrypt`, `--encryption-key`) are ignored for data DBs; per-database encryption is managed via RPC.
-
-## Admin Database (DB 0)
-
-DB 0 acts as the administrative database instance, storing metadata for all user databases (IDs >= 1). It controls existence, access control, and per-database encryption. DB 0 is always encrypted with the `--admin-secret`.
-
-When creating a new database, DB 0 allocates an ID, registers it, and optionally stores a per-database encryption key (write-only). Databases are public by default; use RPC to set them private, requiring access keys for SELECT (read or readwrite based on permissions). Keys are persisted in DB 0 for managed AGE operations.
-
-Access DB 0 with `SELECT 0 KEY <admin-secret>`.
-
-## Symmetric Encryption
-
-HeroDB supports stateless symmetric encryption via SYM commands, using XChaCha20-Poly1305 AEAD.
-
-Commands:
-- `SYM KEYGEN`: Generate 32-byte key. Returns base64-encoded key.
-- `SYM ENCRYPT <key_b64> <message>`: Encrypt message. Returns base64 ciphertext.
-- `SYM DECRYPT <key_b64> <ciphertext_b64>`: Decrypt. Returns plaintext.
-
-Example:
-```bash
-redis-cli SYM KEYGEN
-# → base64_key
-
-redis-cli SYM ENCRYPT base64_key "secret"
-# → base64_ciphertext
-
-redis-cli SYM DECRYPT base64_key base64_ciphertext
-# → "secret"
-```
-
-## RPC Options
-
-Enable the JSON-RPC server with `--enable-rpc` for database management. Methods include creating databases, managing access keys, and setting encryption. See [JSON-RPC Examples](./rpc_examples.md) for payloads.
-
+Here's an expanded version of the cmds.md documentation to include the list commands:
 # HeroDB Commands
 
 HeroDB implements a subset of Redis commands over the Redis protocol. This document describes the available commands and their usage.
@@ -629,29 +575,6 @@ redis-cli -p $PORT AGE LIST
 #    2) "keyname2"
 ```
 
-## SYM Commands
-
-### SYM KEYGEN
-Generate a symmetric encryption key.
-```bash
-redis-cli -p $PORT SYM KEYGEN
-# → base64_encoded_32byte_key
-```
-
-### SYM ENCRYPT
-Encrypt a message with a symmetric key.
-```bash
-redis-cli -p $PORT SYM ENCRYPT <key_b64> "message"
-# → base64_encoded_ciphertext
-```
-
-### SYM DECRYPT
-Decrypt a ciphertext with a symmetric key.
-```bash
-redis-cli -p $PORT SYM DECRYPT <key_b64> <ciphertext_b64>
-# → decrypted_message
-```
-
 ## Server Information Commands
 
 ### INFO
@@ -698,27 +621,3 @@ This expanded documentation includes all the list commands that were implemented
 10. LINDEX - get element by index
 11. LRANGE - get range of elements
 
-
-## Updated Database Selection and Access Keys
-
-HeroDB uses an `Admin DB 0` to control database existence, access, and encryption. Access to data DBs can be public (no key) or private (requires a key). See detailed model in `docs/admin.md`.
-
-Examples:
-
-```bash
-# Public database (no key required)
-redis-cli -p $PORT SELECT 1
-# → OK
-```
-
-```bash
-# Private database (requires access key)
-redis-cli -p $PORT SELECT 2 KEY my-db2-access-key
-# → OK
-```
-
-```bash
-# Admin DB 0 (requires admin secret)
-redis-cli -p $PORT SELECT 0 KEY my-admin-secret
-# → OK
-```

docs/cmds.md

@@ -122,35 +122,4 @@ redis-cli -p 6379 --rdb dump.rdb
 
 # Import to sled
 redis-cli -p 6381 --pipe < dump.rdb
-```
-
-## Authentication and Database Selection
-
-Connections start with no database selected. Any storage-backed command (GET, SET, H*, L*, SCAN, etc.) will return an error until you issue a SELECT to choose a database.
-
-HeroDB uses an `Admin DB 0` to govern database existence, access and per-db encryption. Access control is enforced via `Admin DB 0` metadata. See the full model in (docs/admin.md:1).
-
-Examples:
-```bash
-# Public database (no key required)
-redis-cli -p $PORT SELECT 1
-# → OK
-```
-
-```bash
-# Private database (requires access key)
-redis-cli -p $PORT SELECT 2 KEY my-db2-access-key
-# → OK
-```
-
-```bash
-# Admin DB 0 (requires admin secret)
-redis-cli -p $PORT SELECT 0 KEY my-admin-secret
-# → OK
-```
-
-```bash
-# Before selecting a DB, storage commands will fail
-redis-cli -p $PORT GET key
-# → -ERR No database selected. Use SELECT <id> [KEY <key>] first
 ```

docs/lance.md

@@ -1,444 +0,0 @@
-# Lance Vector Backend (RESP + JSON-RPC)
-
-This document explains how to use HeroDB’s Lance-backed vector store. It is text-first: users provide text, and HeroDB computes embeddings server-side (no manual vectors). It includes copy-pasteable RESP (redis-cli) and JSON-RPC examples for:
-
-- Creating a Lance database
-- Embedding provider configuration (OpenAI, Azure OpenAI, or deterministic test provider)
-- Dataset lifecycle: CREATE, LIST, INFO, DROP
-- Ingestion: STORE text (+ optional metadata)
-- Search: QUERY with K, optional FILTER and RETURN
-- Delete by id
-- Index creation (currently a placeholder/no-op)
-
-References:
-- Implementation: [src/lance_store.rs](src/lance_store.rs), [src/cmd.rs](src/cmd.rs), [src/rpc.rs](src/rpc.rs), [src/server.rs](src/server.rs), [src/embedding.rs](src/embedding.rs)
-
-Notes:
-- Admin DB 0 cannot be Lance (or Tantivy). Only databases with id >= 1 can use Lance.
-- Permissions:
-  - Read operations (SEARCH, LIST, INFO) require read permission.
-  - Mutating operations (CREATE, STORE, CREATEINDEX, DEL, DROP, EMBEDDING CONFIG SET) require readwrite permission.
-- Backend gating:
-  - If a DB is Lance, only LANCE.* and basic control commands (PING, ECHO, SELECT, INFO, CLIENT, etc.) are permitted.
-  - If a DB is not Lance, LANCE.* commands return an error.
-
-Storage layout and schema:
-- Files live at: <base_dir>/lance/<db_id>/<dataset>.lance
-- Records schema:
-  - id: Utf8 (non-null)
-  - vector: FixedSizeList<Float32, dim> (non-null)
-  - text: Utf8 (nullable)
-  - meta: Utf8 JSON (nullable)
-- Search is an L2 KNN brute-force scan for now (lower score = better). Index creation is a no-op placeholder to be implemented later.
-
-Prerequisites:
-- Start HeroDB with RPC enabled (for management calls):
-  - See [docs/basics.md](./basics.md) for flags. Example:
-    ```bash
-    ./target/release/herodb --dir /tmp/herodb --admin-secret mysecret --port 6379 --enable-rpc
-    ```
-
-
-## 0) Create a Lance-backed database (JSON-RPC)
-
-Use the management API to create a database with backend "Lance". DB 0 is reserved for admin and cannot be Lance.
-
-Request:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 1,
-  "method": "herodb_createDatabase",
-  "params": [
-    "Lance",
-    { "name": "vectors-db", "storage_path": null, "max_size": null, "redis_version": null },
-    null
-  ]
-}
-```
-
-- Response contains the allocated db_id (>= 1). Use that id below (replace 1 with your actual id).
-
-Select the database over RESP:
-```bash
-redis-cli -p 6379 SELECT 1
-# → OK
-```
-
-
-## 1) Configure embedding provider (server-side embeddings)
-
-HeroDB embeds text internally at STORE/SEARCH time using a per-dataset EmbeddingConfig sidecar. Configure provider before creating a dataset to choose dimensions and provider.
-
-Supported providers:
-- openai (standard OpenAI or Azure OpenAI)
-- testhash (deterministic, CI-friendly; no network)
-
-Environment variables for OpenAI:
-- Standard OpenAI: export OPENAI_API_KEY=sk-...
-- Azure OpenAI: export AZURE_OPENAI_API_KEY=...
-
-RESP examples:
-```bash
-# Standard OpenAI with default dims (model-dependent, e.g. 1536)
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG SET myset PROVIDER openai MODEL text-embedding-3-small
-
-# OpenAI with reduced output dimension (e.g., 512) when supported
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG SET myset PROVIDER openai MODEL text-embedding-3-small PARAM dim 512
-
-# Azure OpenAI (set env: AZURE_OPENAI_API_KEY)
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG SET myset PROVIDER openai MODEL text-embedding-3-small \
-  PARAM use_azure true \
-  PARAM azure_endpoint https://myresource.openai.azure.com \
-  PARAM azure_deployment my-embed-deploy \
-  PARAM azure_api_version 2024-02-15 \
-  PARAM dim 512
-
-# Deterministic test provider (no network, stable vectors)
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG SET myset PROVIDER testhash MODEL any
-```
-
-Read config:
-```bash
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG GET myset
-# → JSON blob describing provider/model/params
-```
-
-JSON-RPC examples:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 2,
-  "method": "herodb_lanceSetEmbeddingConfig",
-  "params": [
-    1,
-    "myset",
-    "openai",
-    "text-embedding-3-small",
-    { "dim": "512" }
-  ]
-}
-```
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 3,
-  "method": "herodb_lanceGetEmbeddingConfig",
-  "params": [1, "myset"]
-}
-```
-
-
-## 2) Create a dataset
-
-Choose a dimension that matches your embedding configuration. For OpenAI text-embedding-3-small without dimension override, typical dimension is 1536; when `dim` is set (e.g., 512), use that. The current API requires an explicit DIM.
-
-RESP:
-```bash
-redis-cli -p 6379 LANCE.CREATE myset DIM 512
-# → OK
-```
-
-JSON-RPC:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 4,
-  "method": "herodb_lanceCreate",
-  "params": [1, "myset", 512]
-}
-```
-
-
-## 3) Store text documents (server-side embedding)
-
-Provide your id, the text to embed, and optional META fields. The server computes the embedding using the configured provider and stores id/vector/text/meta in the Lance dataset. Upserts by id are supported via delete-then-append semantics.
-
-RESP:
-```bash
-redis-cli -p 6379 LANCE.STORE myset ID doc-1 TEXT "Hello vector world" META title "Hello" category "demo"
-# → OK
-```
-
-JSON-RPC:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 5,
-  "method": "herodb_lanceStoreText",
-  "params": [
-    1,
-    "myset",
-    "doc-1",
-    "Hello vector world",
-    { "title": "Hello", "category": "demo" }
-  ]
-}
-```
-
-
-## 4) Search with a text query
-
-Provide a query string; the server embeds it and performs KNN search. Optional: FILTER expression and RETURN subset of fields.
-
-RESP:
-```bash
-# K nearest neighbors for the query text
-redis-cli -p 6379 LANCE.SEARCH myset K 5 QUERY "greetings to vectors"
-# → Array of hits: [id, score, [k,v, ...]] pairs, lower score = closer
-
-# With a filter on meta fields and return only title
-redis-cli -p 6379 LANCE.SEARCH myset K 3 QUERY "greetings to vectors" FILTER "category = 'demo'" RETURN 1 title
-```
-
-JSON-RPC:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 6,
-  "method": "herodb_lanceSearchText",
-  "params": [1, "myset", "greetings to vectors", 5, null, null]
-}
-```
-
-With filter and selected fields:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 7,
-  "method": "herodb_lanceSearchText",
-  "params": [1, "myset", "greetings to vectors", 3, "category = 'demo'", ["title"]]
-}
-```
-
-Response shape:
-- RESP over redis-cli: an array of hits [id, score, [k, v, ...]].
-- JSON-RPC returns an object containing the RESP-encoded wire format string or a structured result depending on implementation. See [src/rpc.rs](src/rpc.rs) for details.
-
-
-## 5) Create an index (placeholder)
-
-Index creation currently returns OK but is a no-op. It will integrate Lance vector indices in a future update.
-
-RESP:
-```bash
-redis-cli -p 6379 LANCE.CREATEINDEX myset TYPE "ivf_pq" PARAM nlist 100 PARAM pq_m 16
-# → OK (no-op for now)
-```
-
-JSON-RPC:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 8,
-  "method": "herodb_lanceCreateIndex",
-  "params": [1, "myset", "ivf_pq", { "nlist": "100", "pq_m": "16" }]
-}
-```
-
-
-## 6) Inspect datasets
-
-RESP:
-```bash
-# List datasets in current Lance DB
-redis-cli -p 6379 LANCE.LIST
-
-# Get dataset info
-redis-cli -p 6379 LANCE.INFO myset
-```
-
-JSON-RPC:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 9,
-  "method": "herodb_lanceList",
-  "params": [1]
-}
-```
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 10,
-  "method": "herodb_lanceInfo",
-  "params": [1, "myset"]
-}
-```
-
-
-## 7) Delete and drop
-
-RESP:
-```bash
-# Delete by id
-redis-cli -p 6379 LANCE.DEL myset doc-1
-# → OK
-
-# Drop the entire dataset
-redis-cli -p 6379 LANCE.DROP myset
-# → OK
-```
-
-JSON-RPC:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 11,
-  "method": "herodb_lanceDel",
-  "params": [1, "myset", "doc-1"]
-}
-```
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 12,
-  "method": "herodb_lanceDrop",
-  "params": [1, "myset"]
-}
-```
-
-
-## 8) End-to-end example (RESP)
-
-```bash
-# 1. Select Lance DB (assume db_id=1 created via RPC)
-redis-cli -p 6379 SELECT 1
-
-# 2. Configure embedding provider (OpenAI small model at 512 dims)
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG SET myset PROVIDER openai MODEL text-embedding-3-small PARAM dim 512
-
-# 3. Create dataset
-redis-cli -p 6379 LANCE.CREATE myset DIM 512
-
-# 4. Store documents
-redis-cli -p 6379 LANCE.STORE myset ID doc-1 TEXT "The quick brown fox jumps over the lazy dog" META title "Fox" category "animal"
-redis-cli -p 6379 LANCE.STORE myset ID doc-2 TEXT "A fast auburn fox vaulted a sleepy canine" META title "Fox paraphrase" category "animal"
-
-# 5. Search
-redis-cli -p 6379 LANCE.SEARCH myset K 2 QUERY "quick brown fox" RETURN 1 title
-
-# 6. Dataset info and listing
-redis-cli -p 6379 LANCE.INFO myset
-redis-cli -p 6379 LANCE.LIST
-
-# 7. Delete and drop
-redis-cli -p 6379 LANCE.DEL myset doc-2
-redis-cli -p 6379 LANCE.DROP myset
-```
-
-
-## 9) End-to-end example (JSON-RPC)
-
-Assume RPC server on port 8080. Replace ids and ports as needed.
-
-1) Create Lance DB:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 100,
-  "method": "herodb_createDatabase",
-  "params": ["Lance", { "name": "vectors-db", "storage_path": null, "max_size": null, "redis_version": null }, null]
-}
-```
-
-2) Set embedding config:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 101,
-  "method": "herodb_lanceSetEmbeddingConfig",
-  "params": [1, "myset", "openai", "text-embedding-3-small", { "dim": "512" }]
-}
-```
-
-3) Create dataset:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 102,
-  "method": "herodb_lanceCreate",
-  "params": [1, "myset", 512]
-}
-```
-
-4) Store text:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 103,
-  "method": "herodb_lanceStoreText",
-  "params": [1, "myset", "doc-1", "The quick brown fox jumps over the lazy dog", { "title": "Fox", "category": "animal" }]
-}
-```
-
-5) Search text:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 104,
-  "method": "herodb_lanceSearchText",
-  "params": [1, "myset", "quick brown fox", 2, null, ["title"]]
-}
-```
-
-6) Info/list:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 105,
-  "method": "herodb_lanceInfo",
-  "params": [1, "myset"]
-}
-```
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 106,
-  "method": "herodb_lanceList",
-  "params": [1]
-}
-```
-
-7) Delete/drop:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 107,
-  "method": "herodb_lanceDel",
-  "params": [1, "myset", "doc-1"]
-}
-```
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 108,
-  "method": "herodb_lanceDrop",
-  "params": [1, "myset"]
-}
-```
-
-
-## 10) Operational notes and troubleshooting
-
-- If using OpenAI and you see “missing API key env”, set:
-  - Standard: `export OPENAI_API_KEY=sk-...`
-  - Azure: `export AZURE_OPENAI_API_KEY=...` and pass `use_azure true`, `azure_endpoint`, `azure_deployment`, `azure_api_version`.
-- Dimensions mismatch:
-  - Ensure the dataset DIM equals the provider’s embedding dim. For OpenAI text-embedding-3 models, set `PARAM dim 512` (or another supported size) and use that same DIM for `LANCE.CREATE`.
-- DB 0 restriction:
-  - Lance is not allowed on DB 0. Use db_id >= 1.
-- Permissions:
-  - Read operations (SEARCH, LIST, INFO) require read permission.
-  - Mutations (CREATE, STORE, CREATEINDEX, DEL, DROP, EMBEDDING CONFIG SET) require readwrite permission.
-- Backend gating:
-  - On Lance DBs, only LANCE.* commands are accepted (plus basic control).
-- Current index behavior:
-  - `LANCE.CREATEINDEX` returns OK but is a no-op. Future versions will integrate Lance vector indices.
-- Implementation files for reference:
-  - [src/lance_store.rs](src/lance_store.rs), [src/cmd.rs](src/cmd.rs), [src/rpc.rs](src/rpc.rs), [src/server.rs](src/server.rs), [src/embedding.rs](src/embedding.rs)

docs/lance_vector_db.md

@@ -0,0 +1,454 @@
+# Lance Vector Database Operations
+
+HeroDB includes a powerful vector database integration using Lance, enabling high-performance vector storage, search, and multimodal data management. By default, it uses Ollama for local text embeddings, with support for custom external embedding services.
+
+## Overview
+
+The Lance vector database integration provides:
+
+- **High-performance vector storage** using Lance's columnar format
+- **Local Ollama integration** for text embeddings (default, no external dependencies)
+- **Custom embedding service support** for advanced use cases
+- **Text embedding support** (images via custom services)
+- **Vector similarity search** with configurable parameters
+- **Scalable indexing** with IVF_PQ (Inverted File with Product Quantization)
+- **Redis-compatible command interface**
+
+## Architecture
+
+```
+┌─────────────────┐    ┌──────────────────┐    ┌─────────────────┐
+│   HeroDB        │    │  External        │    │   Lance         │
+│   Redis Server  │◄──►│  Embedding       │    │   Vector Store  │
+│                 │    │  Service         │    │                 │
+└─────────────────┘    └──────────────────┘    └─────────────────┘
+         │                       │                       │
+         │                       │                       │
+    Redis Protocol          HTTP API              Arrow/Parquet
+    Commands                JSON Requests         Columnar Storage
+```
+
+### Key Components
+
+1. **Lance Store**: High-performance columnar vector storage
+2. **Ollama Integration**: Local embedding service (default)
+3. **Custom Embedding Service**: Optional HTTP API for advanced use cases
+4. **Redis Command Interface**: Familiar Redis-style commands
+5. **Arrow Schema**: Flexible schema definition for metadata
+
+## Configuration
+
+### Default Setup (Ollama)
+
+HeroDB uses Ollama by default for text embeddings. No configuration is required if Ollama is running locally:
+
+```bash
+# Install Ollama (if not already installed)
+# Visit: https://ollama.ai
+
+# Pull the embedding model
+ollama pull nomic-embed-text
+
+# Ollama automatically runs on localhost:11434
+# HeroDB will use this by default
+```
+
+**Default Configuration:**
+- **URL**: `http://localhost:11434`
+- **Model**: `nomic-embed-text`
+- **Dimensions**: 768 (for nomic-embed-text)
+
+### Custom Embedding Service (Optional)
+
+To use a custom embedding service instead of Ollama:
+
+```bash
+# Set custom embedding service URL
+redis-cli HSET config:core:aiembed url "http://your-embedding-service:8080/embed"
+
+# Optional: Set authentication if required
+redis-cli HSET config:core:aiembed token "your-api-token"
+```
+
+### Embedding Service API Contracts
+
+#### Ollama API (Default)
+HeroDB calls Ollama using this format:
+
+```bash
+POST http://localhost:11434/api/embeddings
+Content-Type: application/json
+
+{
+  "model": "nomic-embed-text",
+  "prompt": "Your text to embed"
+}
+```
+
+Response:
+```json
+{
+  "embedding": [0.1, 0.2, 0.3, ...]
+}
+```
+
+#### Custom Service API
+Your custom embedding service should accept POST requests with this JSON format:
+
+```json
+{
+  "texts": ["text1", "text2"],           // Optional: array of texts
+  "images": ["base64_image1", "base64_image2"], // Optional: base64 encoded images
+  "model": "your-model-name"             // Optional: model specification
+}
+```
+
+And return responses in this format:
+
+```json
+{
+  "embeddings": [[0.1, 0.2, ...], [0.3, 0.4, ...]],  // Array of embedding vectors
+  "model": "model-name",                               // Model used
+  "usage": {                                          // Optional usage stats
+    "tokens": 100,
+    "requests": 2
+  }
+}
+```
+
+## Commands Reference
+
+### Dataset Management
+
+#### LANCE CREATE
+Create a new vector dataset with specified dimensions and optional schema.
+
+```bash
+LANCE CREATE <dataset> DIM <dimension> [SCHEMA field:type ...]
+```
+
+**Parameters:**
+- `dataset`: Name of the dataset
+- `dimension`: Vector dimension (e.g., 384, 768, 1536)
+- `field:type`: Optional metadata fields (string, int, float, bool)
+
+**Examples:**
+```bash
+# Create a simple dataset for 384-dimensional vectors
+LANCE CREATE documents DIM 384
+
+# Create dataset with metadata schema
+LANCE CREATE products DIM 768 SCHEMA category:string price:float available:bool
+```
+
+#### LANCE LIST
+List all available datasets.
+
+```bash
+LANCE LIST
+```
+
+**Returns:** Array of dataset names
+
+#### LANCE INFO
+Get information about a specific dataset.
+
+```bash
+LANCE INFO <dataset>
+```
+
+**Returns:** Dataset metadata including name, version, row count, and schema
+
+#### LANCE DROP
+Delete a dataset and all its data.
+
+```bash
+LANCE DROP <dataset>
+```
+
+### Data Operations
+
+#### LANCE STORE
+Store multimodal data (text/images) with automatic embedding generation.
+
+```bash
+LANCE STORE <dataset> [TEXT <text>] [IMAGE <base64>] [key value ...]
+```
+
+**Parameters:**
+- `dataset`: Target dataset name
+- `TEXT`: Text content to embed
+- `IMAGE`: Base64-encoded image to embed
+- `key value`: Metadata key-value pairs
+
+**Examples:**
+```bash
+# Store text with metadata
+LANCE STORE documents TEXT "Machine learning is transforming industries" category "AI" author "John Doe"
+
+# Store image with metadata
+LANCE STORE images IMAGE "iVBORw0KGgoAAAANSUhEUgAA..." category "nature" tags "landscape,mountains"
+
+# Store both text and image
+LANCE STORE multimodal TEXT "Beautiful sunset" IMAGE "base64data..." location "California"
+```
+
+**Returns:** Unique ID of the stored item
+
+### Search Operations
+
+#### LANCE SEARCH
+Search using a raw vector.
+
+```bash
+LANCE SEARCH <dataset> VECTOR <vector> K <k> [NPROBES <n>] [REFINE <r>]
+```
+
+**Parameters:**
+- `dataset`: Dataset to search
+- `vector`: Comma-separated vector values (e.g., "0.1,0.2,0.3")
+- `k`: Number of results to return
+- `NPROBES`: Number of partitions to search (optional)
+- `REFINE`: Refine factor for better accuracy (optional)
+
+**Example:**
+```bash
+LANCE SEARCH documents VECTOR "0.1,0.2,0.3,0.4" K 5 NPROBES 10
+```
+
+#### LANCE SEARCH.TEXT
+Search using text query (automatically embedded).
+
+```bash
+LANCE SEARCH.TEXT <dataset> <query_text> K <k> [NPROBES <n>] [REFINE <r>]
+```
+
+**Parameters:**
+- `dataset`: Dataset to search
+- `query_text`: Text query to search for
+- `k`: Number of results to return
+- `NPROBES`: Number of partitions to search (optional)
+- `REFINE`: Refine factor for better accuracy (optional)
+
+**Example:**
+```bash
+LANCE SEARCH.TEXT documents "artificial intelligence applications" K 10 NPROBES 20
+```
+
+**Returns:** Array of results with distance scores and metadata
+
+### Embedding Operations
+
+#### LANCE EMBED.TEXT
+Generate embeddings for text without storing.
+
+```bash
+LANCE EMBED.TEXT <text1> [text2] [text3] ...
+```
+
+**Example:**
+```bash
+LANCE EMBED.TEXT "Hello world" "Machine learning" "Vector database"
+```
+
+**Returns:** Array of embedding vectors
+
+### Index Management
+
+#### LANCE CREATE.INDEX
+Create a vector index for faster search performance.
+
+```bash
+LANCE CREATE.INDEX <dataset> <index_type> [PARTITIONS <n>] [SUBVECTORS <n>]
+```
+
+**Parameters:**
+- `dataset`: Dataset to index
+- `index_type`: Index type (currently supports "IVF_PQ")
+- `PARTITIONS`: Number of partitions (default: 256)
+- `SUBVECTORS`: Number of sub-vectors for PQ (default: 16)
+
+**Example:**
+```bash
+LANCE CREATE.INDEX documents IVF_PQ PARTITIONS 512 SUBVECTORS 32
+```
+
+## Usage Patterns
+
+### 1. Document Search System
+
+```bash
+# Setup
+LANCE CREATE documents DIM 384 SCHEMA title:string content:string category:string
+
+# Store documents
+LANCE STORE documents TEXT "Introduction to machine learning algorithms" title "ML Basics" category "education"
+LANCE STORE documents TEXT "Deep learning neural networks explained" title "Deep Learning" category "education"
+LANCE STORE documents TEXT "Building scalable web applications" title "Web Dev" category "programming"
+
+# Create index for better performance
+LANCE CREATE.INDEX documents IVF_PQ PARTITIONS 256
+
+# Search
+LANCE SEARCH.TEXT documents "neural networks" K 5
+```
+
+### 2. Image Similarity Search
+
+```bash
+# Setup
+LANCE CREATE images DIM 512 SCHEMA filename:string tags:string
+
+# Store images (base64 encoded)
+LANCE STORE images IMAGE "iVBORw0KGgoAAAANSUhEUgAA..." filename "sunset.jpg" tags "nature,landscape"
+LANCE STORE images IMAGE "iVBORw0KGgoAAAANSUhEUgBB..." filename "city.jpg" tags "urban,architecture"
+
+# Search by image
+LANCE STORE temp_search IMAGE "query_image_base64..."
+# Then use the returned ID to get embedding and search
+```
+
+### 3. Multimodal Content Management
+
+```bash
+# Setup
+LANCE CREATE content DIM 768 SCHEMA type:string source:string
+
+# Store mixed content
+LANCE STORE content TEXT "Product description for smartphone" type "product" source "catalog"
+LANCE STORE content IMAGE "product_image_base64..." type "product_image" source "catalog"
+
+# Search across all content types
+LANCE SEARCH.TEXT content "smartphone features" K 10
+```
+
+## Performance Considerations
+
+### Vector Dimensions
+- **384**: Good for general text (e.g., sentence-transformers)
+- **768**: Standard for BERT-like models
+- **1536**: OpenAI text-embedding-ada-002
+- **Higher dimensions**: Better accuracy but slower search
+
+### Index Configuration
+- **More partitions**: Better for larger datasets (>100K vectors)
+- **More sub-vectors**: Better compression but slower search
+- **NPROBES**: Higher values = better accuracy, slower search
+
+### Best Practices
+
+1. **Create indexes** for datasets with >1000 vectors
+2. **Use appropriate dimensions** based on your embedding model
+3. **Configure NPROBES** based on accuracy vs speed requirements
+4. **Batch operations** when possible for better performance
+5. **Monitor embedding service** response times and rate limits
+
+## Error Handling
+
+Common error scenarios and solutions:
+
+### Embedding Service Errors
+```bash
+# Error: Embedding service not configured
+ERR Embedding service URL not configured. Set it with: HSET config:core:aiembed url <YOUR_EMBEDDING_SERVICE_URL>
+
+# Error: Service unavailable
+ERR Embedding service returned error 404 Not Found
+```
+
+**Solution:** Ensure embedding service is running and URL is correct.
+
+### Dataset Errors
+```bash
+# Error: Dataset doesn't exist
+ERR Dataset 'mydata' does not exist
+
+# Error: Dimension mismatch
+ERR Vector dimension mismatch: expected 384, got 768
+```
+
+**Solution:** Create dataset first or check vector dimensions.
+
+### Search Errors
+```bash
+# Error: Invalid vector format
+ERR Invalid vector format
+
+# Error: No index available
+ERR No index available for fast search
+```
+
+**Solution:** Check vector format or create an index.
+
+## Integration Examples
+
+### With Python
+```python
+import redis
+import json
+
+r = redis.Redis(host='localhost', port=6379)
+
+# Create dataset
+r.execute_command('LANCE', 'CREATE', 'docs', 'DIM', '384')
+
+# Store document
+result = r.execute_command('LANCE', 'STORE', 'docs', 
+                          'TEXT', 'Machine learning tutorial',
+                          'category', 'education')
+print(f"Stored with ID: {result}")
+
+# Search
+results = r.execute_command('LANCE', 'SEARCH.TEXT', 'docs', 
+                           'machine learning', 'K', '5')
+print(f"Search results: {results}")
+```
+
+### With Node.js
+```javascript
+const redis = require('redis');
+const client = redis.createClient();
+
+// Create dataset
+await client.sendCommand(['LANCE', 'CREATE', 'docs', 'DIM', '384']);
+
+// Store document
+const id = await client.sendCommand(['LANCE', 'STORE', 'docs', 
+                                   'TEXT', 'Deep learning guide',
+                                   'category', 'AI']);
+
+// Search
+const results = await client.sendCommand(['LANCE', 'SEARCH.TEXT', 'docs',
+                                        'deep learning', 'K', '10']);
+```
+
+## Monitoring and Maintenance
+
+### Health Checks
+```bash
+# Check if Lance store is available
+LANCE LIST
+
+# Check dataset health
+LANCE INFO mydataset
+
+# Test embedding service
+LANCE EMBED.TEXT "test"
+```
+
+### Maintenance Operations
+```bash
+# Backup: Use standard Redis backup procedures
+# The Lance data is stored separately in the data directory
+
+# Cleanup: Remove unused datasets
+LANCE DROP old_dataset
+
+# Reindex: Drop and recreate indexes if needed
+LANCE DROP dataset_name
+LANCE CREATE dataset_name DIM 384
+# Re-import data
+LANCE CREATE.INDEX dataset_name IVF_PQ
+```
+
+This integration provides a powerful foundation for building AI-powered applications with vector search capabilities while maintaining the familiar Redis interface.

docs/lancedb_text_and_images_example.md

@@ -1,138 +0,0 @@
-# LanceDB Text and Images: End-to-End Example
-
-This guide demonstrates creating a Lance backend database, ingesting two text documents and two images, performing searches over both, and cleaning up the datasets.
-
-Prerequisites
-- Build HeroDB and start the server with JSON-RPC enabled.
-Commands:
-```bash
-cargo build --release
-./target/release/herodb --dir /tmp/herodb --admin-secret mysecret --port 6379 --enable-rpc
-```
-
-We'll use:
-- redis-cli for RESP commands against port 6379
-- curl for JSON-RPC against 8080 if desired
-- Deterministic local embedders to avoid external dependencies: testhash (text, dim 64) and testimagehash (image, dim 512)
-
-0) Create a Lance-backed database (JSON-RPC)
-Request:
-```json
-{ "jsonrpc": "2.0", "id": 1, "method": "herodb_createDatabase", "params": ["Lance", { "name": "media-db", "storage_path": null, "max_size": null, "redis_version": null }, null] }
-```
-Response returns db_id (assume 1). Select DB over RESP:
-```bash
-redis-cli -p 6379 SELECT 1
-# → OK
-```
-
-1) Configure embedding providers
-We'll create two datasets with independent embedding configs:
-- textset → provider testhash, dim 64
-- imageset → provider testimagehash, dim 512
-
-Text config:
-```bash
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG SET textset PROVIDER testhash MODEL any PARAM dim 64
-# → OK
-```
-Image config:
-```bash
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG SET imageset PROVIDER testimagehash MODEL any PARAM dim 512
-# → OK
-```
-
-2) Create datasets
-```bash
-redis-cli -p 6379 LANCE.CREATE textset DIM 64
-# → OK
-redis-cli -p 6379 LANCE.CREATE imageset DIM 512
-# → OK
-```
-
-3) Ingest two text documents (server-side embedding)
-```bash
-redis-cli -p 6379 LANCE.STORE textset ID doc-1 TEXT "The quick brown fox jumps over the lazy dog" META title "Fox" category "animal"
-# → OK
-redis-cli -p 6379 LANCE.STORE textset ID doc-2 TEXT "A fast auburn fox vaulted a sleepy canine" META title "Paraphrase" category "animal"
-# → OK
-```
-
-4) Ingest two images
-You can provide a URI or base64 bytes. Use URI for URIs, BYTES for base64 data.
-Example using free placeholder images:
-```bash
-# Store via URI
-redis-cli -p 6379 LANCE.STOREIMAGE imageset ID img-1 URI "https://picsum.photos/seed/1/256/256" META title "Seed1" group "demo"
-# → OK
-redis-cli -p 6379 LANCE.STOREIMAGE imageset ID img-2 URI "https://picsum.photos/seed/2/256/256" META title "Seed2" group "demo"
-# → OK
-```
-If your environment blocks outbound HTTP, you can embed image bytes:
-```bash
-# Example: read a local file and base64 it (replace path)
-b64=$(base64 -w0 ./image1.png)
-redis-cli -p 6379 LANCE.STOREIMAGE imageset ID img-b64-1 BYTES "$b64" META title "Local1" group "demo"
-```
-
-5) Search text
-```bash
-# Top-2 nearest neighbors for a query
-redis-cli -p 6379 LANCE.SEARCH textset K 2 QUERY "quick brown fox" RETURN 1 title
-# → 1) [id, score, [k1,v1,...]]
-```
-With a filter (supports equality on schema or meta keys):
-```bash
-redis-cli -p 6379 LANCE.SEARCH textset K 2 QUERY "fox jumps" FILTER "category = 'animal'" RETURN 1 title
-```
-
-6) Search images
-```bash
-# Provide a URI as the query
-redis-cli -p 6379 LANCE.SEARCHIMAGE imageset K 2 QUERYURI "https://picsum.photos/seed/1/256/256" RETURN 1 title
-
-# Or provide base64 bytes as the query
-qb64=$(curl -s https://picsum.photos/seed/3/256/256 | base64 -w0)
-redis-cli -p 6379 LANCE.SEARCHIMAGE imageset K 2 QUERYBYTES "$qb64" RETURN 1 title
-```
-
-7) Inspect datasets
-```bash
-redis-cli -p 6379 LANCE.LIST
-redis-cli -p 6379 LANCE.INFO textset
-redis-cli -p 6379 LANCE.INFO imageset
-```
-
-8) Delete by id and drop datasets
-```bash
-# Delete one record
-redis-cli -p 6379 LANCE.DEL textset doc-2
-# → OK
-
-# Drop entire datasets
-redis-cli -p 6379 LANCE.DROP textset
-redis-cli -p 6379 LANCE.DROP imageset
-# → OK
-```
-
-Appendix: Using OpenAI embeddings instead of test providers
-Text:
-```bash
-export OPENAI_API_KEY=sk-...
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG SET textset PROVIDER openai MODEL text-embedding-3-small PARAM dim 512
-redis-cli -p 6379 LANCE.CREATE textset DIM 512
-```
-Azure OpenAI:
-```bash
-export AZURE_OPENAI_API_KEY=...
-redis-cli -p 6379 LANCE.EMBEDDING CONFIG SET textset PROVIDER openai MODEL text-embedding-3-small \
-  PARAM use_azure true \
-  PARAM azure_endpoint https://myresource.openai.azure.com \
-  PARAM azure_deployment my-embed-deploy \
-  PARAM azure_api_version 2024-02-15 \
-  PARAM dim 512
-```
-Notes:
-- Ensure dataset DIM matches the configured embedding dimension.
-- Lance is only available for non-admin databases (db_id >= 1).
-- On Lance DBs, only LANCE.* and basic control commands are allowed.

docs/rpc_examples.md

@@ -1,141 +0,0 @@
-# HeroDB JSON-RPC Examples
-
-These examples show full JSON-RPC 2.0 payloads for managing HeroDB via the RPC API (enable with `--enable-rpc`). Methods are named as `hero_<function>`. Params are positional arrays; enum values are strings (e.g., `"Redb"`). Copy-paste into Postman or similar clients.
-
-## Database Management
-
-### Create Database
-Creates a new database with optional per-database encryption key (stored write-only in Admin DB 0).
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 1,
-  "method": "hero_createDatabase",
-  "params": [
-    "Redb",
-    { "name": null, "storage_path": null, "max_size": null, "redis_version": null },
-    null
-  ]
-}
-```
-
-With encryption:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 2,
-  "method": "hero_createDatabase",
-  "params": [
-    "Sled",
-    { "name": "secure-db", "storage_path": null, "max_size": null, "redis_version": null },
-    "my-per-db-encryption-key"
-  ]
-}
-```
-
-### List Databases
-Returns array of database infos (id, backend, encrypted status, size, etc.).
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 3,
-  "method": "hero_listDatabases",
-  "params": []
-}
-```
-
-### Get Database Info
-Retrieves detailed info for a specific database.
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 4,
-  "method": "hero_getDatabaseInfo",
-  "params": [1]
-}
-```
-
-### Delete Database
-Removes physical database file; metadata remains in Admin DB 0.
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 5,
-  "method": "hero_deleteDatabase",
-  "params": [1]
-}
-```
-
-## Access Control
-
-### Add Access Key
-Adds a hashed access key for private databases. Permissions: `"read"` or `"readwrite"`.
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 6,
-  "method": "hero_addAccessKey",
-  "params": [2, "my-access-key", "readwrite"]
-}
-```
-
-### List Access Keys
-Returns array of key hashes, permissions, and creation timestamps.
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 7,
-  "method": "hero_listAccessKeys",
-  "params": [2]
-}
-```
-
-### Delete Access Key
-Removes key by its SHA-256 hash.
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 8,
-  "method": "hero_deleteAccessKey",
-  "params": [2, "0123abcd...keyhash..."]
-}
-```
-
-### Set Database Public/Private
-Toggles public access (default true). Private databases require access keys.
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 9,
-  "method": "hero_setDatabasePublic",
-  "params": [2, false]
-}
-```
-
-## Server Info
-
-### Get Server Stats
-Returns stats like total databases and uptime.
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 10,
-  "method": "hero_getServerStats",
-  "params": []
-}
-```
-
-## Notes
-- Per-database encryption keys are write-only; set at creation and used transparently.
-- Access keys are hashed (SHA-256) for storage; provide plaintext in requests.
-- Backend options: `"Redb"` (default) or `"Sled"`.
-- Config object fields (name, storage_path, etc.) are optional and currently ignored but positional.

docs/tantivy.md

@@ -1,253 +0,0 @@
-# Tantivy Full‑Text Backend (JSON‑RPC)
-
-This document explains how to use HeroDB’s Tantivy-backed full‑text search as a dedicated database backend and provides copy‑pasteable JSON‑RPC requests. Tantivy is available only for non‑admin databases (db_id >= 1). Admin DB 0 always uses Redb/Sled and rejects FT operations.
-
-Important characteristics:
-- Tantivy is a third backend alongside Redb and Sled. It provides search indexes only; there is no KV store backing it.
-- On Tantivy databases, Redis KV/list/hash commands are rejected; only FT commands and basic control (SELECT, CLIENT, INFO, etc.) are allowed.
-- FT JSON‑RPC is namespaced as "herodb" and methods are named with underscore: herodb_ftCreate, herodb_ftAdd, herodb_ftSearch, herodb_ftDel, herodb_ftInfo, herodb_ftDrop.
-
-Reference to server implementation:
-- RPC methods are defined in [rust.trait Rpc()](src/rpc.rs:70):
-  - [rust.fn ft_create()](src/rpc.rs:121)
-  - [rust.fn ft_add()](src/rpc.rs:130)
-  - [rust.fn ft_search()](src/rpc.rs:141)
-  - [rust.fn ft_del()](src/rpc.rs:154)
-  - [rust.fn ft_info()](src/rpc.rs:158)
-  - [rust.fn ft_drop()](src/rpc.rs:162)
-
-Notes on responses:
-- ftCreate/ftAdd/ftDel/ftDrop return a JSON boolean: true on success.
-- ftSearch/ftInfo return a JSON object with a single key "resp" containing a RESP‑encoded string (wire format used by Redis). You can display or parse it on the client side as needed.
-
-RESP usage (redis-cli):
-- For RESP clients, you must SELECT the Tantivy database first. SELECT now succeeds for Tantivy DBs without opening KV storage.
-- After SELECT, you can run FT.* commands within that DB context.
-
-Example with redis-cli:
-```bash
-# Connect to server
-redis-cli -p 6379
-
-# Select Tantivy DB 1 (public by default)
-SELECT 1
-# → OK
-
-# Create index
-FT.CREATE product_catalog SCHEMA title TEXT description TEXT category TAG price NUMERIC rating NUMERIC location GEO
-# → OK
-
-# Add a document
-FT.ADD product_catalog product:1 1.0 title "Wireless Bluetooth Headphones" description "Premium noise-canceling headphones with 30-hour battery life" category "electronics,audio" price 299.99 rating 4.5 location "-122.4194,37.7749"
-# → OK
-
-# Search
-FT.SEARCH product_catalog wireless LIMIT 0 3
-# → RESP array with hits
-```
-
-Storage layout (on disk):
-- Indices are stored per database under:
-  - <base_dir>/search_indexes/<db_id>/<index_name>
-- Example: /tmp/test/search_indexes/1/product_catalog
-
-0) Create a new Tantivy database
-
-Use herodb_createDatabase with backend "Tantivy". DB 0 cannot be Tantivy.
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 1,
-  "method": "herodb_createDatabase",
-  "params": [
-    "Tantivy",
-    { "name": "search-db", "storage_path": null, "max_size": null, "redis_version": null },
-    null
-  ]
-}
-```
-
-The response contains the allocated db_id (>= 1). Use that id in the calls below.
-
-1) FT.CREATE — create an index with schema
-
-Method: herodb_ftCreate → [rust.fn ft_create()](src/rpc.rs:121)
-
-Schema format is an array of tuples: [ [field_name, field_type, [options...] ], ... ]
-Supported field types: "TEXT", "NUMERIC" (defaults to F64), "TAG", "GEO"
-Supported options (subset): "WEIGHT", "SORTABLE", "NOINDEX", "SEPARATOR", "CASESENSITIVE"
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 2,
-  "method": "herodb_ftCreate",
-  "params": [
-    1,
-    "product_catalog",
-    [
-      ["title", "TEXT", ["SORTABLE"]],
-      ["description", "TEXT", []],
-      ["category", "TAG", ["SEPARATOR", ","]],
-      ["price", "NUMERIC", ["SORTABLE"]],
-      ["rating", "NUMERIC", []],
-      ["location", "GEO", []]
-    ]
-  ]
-}
-```
-
-Returns: true on success.
-
-2) FT.ADD — add or replace a document
-
-Method: herodb_ftAdd → [rust.fn ft_add()](src/rpc.rs:130)
-
-Fields is an object (map) of field_name → value (all values are sent as strings). GEO expects "lat,lon".
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 3,
-  "method": "herodb_ftAdd",
-  "params": [
-    1,
-    "product_catalog",
-    "product:1",
-    1.0,
-    {
-      "title": "Wireless Bluetooth Headphones",
-      "description": "Premium noise-canceling headphones with 30-hour battery life",
-      "category": "electronics,audio",
-      "price": "299.99",
-      "rating": "4.5",
-      "location": "-122.4194,37.7749"
-    }
-  ]
-}
-```
-
-Returns: true on success.
-
-3) FT.SEARCH — query an index
-
-Method: herodb_ftSearch → [rust.fn ft_search()](src/rpc.rs:141)
-
-Parameters: (db_id, index_name, query, filters?, limit?, offset?, return_fields?)
-- filters: array of [field, value] pairs (Equals filter)
-- limit/offset: numbers (defaults: limit=10, offset=0)
-- return_fields: array of field names to include (optional)
-
-Simple query:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 4,
-  "method": "herodb_ftSearch",
-  "params": [1, "product_catalog", "wireless", null, 10, 0, null]
-}
-```
-
-Pagination + filters + selected fields:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 5,
-  "method": "herodb_ftSearch",
-  "params": [
-    1,
-    "product_catalog",
-    "mouse",
-    [["category", "electronics"]],
-    5,
-    0,
-    ["title", "price", "rating"]
-  ]
-}
-```
-
-Response shape:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 5,
-  "result": { "resp": "*...RESP encoded array..." }
-}
-```
-
-4) FT.INFO — index metadata
-
-Method: herodb_ftInfo → [rust.fn ft_info()](src/rpc.rs:158)
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 6,
-  "method": "herodb_ftInfo",
-  "params": [1, "product_catalog"]
-}
-```
-
-Response shape:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 6,
-  "result": { "resp": "*...RESP encoded array with fields and counts..." }
-}
-```
-
-5) FT.DEL — delete by doc id
-
-Method: herodb_ftDel → [rust.fn ft_del()](src/rpc.rs:154)
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 7,
-  "method": "herodb_ftDel",
-  "params": [1, "product_catalog", "product:1"]
-}
-```
-
-Returns: true on success. Note: current implementation logs and returns success; physical delete may be a no‑op until delete is finalized in the engine.
-
-6) FT.DROP — drop an index
-
-Method: herodb_ftDrop → [rust.fn ft_drop()](src/rpc.rs:162)
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 8,
-  "method": "herodb_ftDrop",
-  "params": [1, "product_catalog"]
-}
-```
-
-Returns: true on success.
-
-Field types and options
-
-- TEXT: stored/indexed/tokenized text. "SORTABLE" marks it fast (stored + fast path in our wrapper).
-- NUMERIC: stored/indexed numeric; default precision F64. "SORTABLE" enables fast column.
-- TAG: exact matching terms. Options: "SEPARATOR" (default ","), "CASESENSITIVE" (default false).
-- GEO: "lat,lon" string; stored as two numeric fields internally.
-
-Backend and permission gating
-
-- FT methods are rejected on DB 0.
-- FT methods require the database backend to be Tantivy; otherwise RPC returns an error.
-- Write‑like FT methods (create/add/del/drop) follow the same permission model as Redis writes on selected databases.
-
-Troubleshooting
-
-- "DB backend is not Tantivy": ensure the database was created with backend "Tantivy".
-- "FT not allowed on DB 0": use a non‑admin database id (>= 1).
-- Empty search results: confirm that the queried fields are tokenized/indexed (TEXT) and that documents were added successfully.
-
-Related docs
-
-- Command‑level search overview: [docs/search.md](docs/search.md:1)
-- RPC definitions: [src/rpc.rs](src/rpc.rs:1)

examples/README.md

@@ -1,6 +1,191 @@
-# HeroDB Tantivy Search Examples
+# HeroDB Examples
 
-This directory contains examples demonstrating HeroDB's full-text search capabilities powered by Tantivy.
+This directory contains examples demonstrating HeroDB's capabilities including full-text search powered by Tantivy and vector database operations using Lance.
+
+## Available Examples
+
+1. **[Tantivy Search Demo](#tantivy-search-demo-bash-script)** - Full-text search capabilities
+2. **[Lance Vector Database Demo](#lance-vector-database-demo-bash-script)** - Vector database and AI operations
+3. **[AGE Encryption Demo](age_bash_demo.sh)** - Cryptographic operations
+4. **[Simple Demo](simple_demo.sh)** - Basic Redis operations
+
+---
+
+## Lance Vector Database Demo (Bash Script)
+
+### Overview
+The `lance_vector_demo.sh` script provides a comprehensive demonstration of HeroDB's vector database capabilities using Lance. It showcases vector storage, similarity search, multimodal data handling, and AI-powered operations with external embedding services.
+
+### Prerequisites
+1. **HeroDB Server**: The server must be running (default port 6379)
+2. **Redis CLI**: The `redis-cli` tool must be installed and available in your PATH
+3. **Embedding Service** (optional): For full functionality, set up an external embedding service
+
+### Running the Demo
+
+#### Step 1: Start HeroDB Server
+```bash
+# From the project root directory
+cargo run -- --dir ./test_data --port 6379
+```
+
+#### Step 2: Run the Demo (in a new terminal)
+```bash
+# From the project root directory
+./examples/lance_vector_demo.sh
+```
+
+### What the Demo Covers
+
+The script demonstrates comprehensive vector database operations:
+
+1. **Dataset Management**
+   - Creating vector datasets with custom dimensions
+   - Defining schemas with metadata fields
+   - Listing and inspecting datasets
+   - Dataset information and statistics
+
+2. **Embedding Operations**
+   - Text embedding generation via external services
+   - Multimodal embedding support (text + images)
+   - Batch embedding operations
+
+3. **Data Storage**
+   - Storing text documents with automatic embedding
+   - Storing images with metadata
+   - Multimodal content storage
+   - Rich metadata support
+
+4. **Vector Search**
+   - Similarity search with raw vectors
+   - Text-based semantic search
+   - Configurable search parameters (K, NPROBES, REFINE)
+   - Cross-modal search capabilities
+
+5. **Index Management**
+   - Creating IVF_PQ indexes for performance
+   - Custom index parameters
+   - Performance optimization
+
+6. **Advanced Features**
+   - Error handling and recovery
+   - Performance testing concepts
+   - Monitoring and maintenance
+   - Cleanup operations
+
+### Key Lance Commands Demonstrated
+
+#### Dataset Management
+```bash
+# Create vector dataset
+LANCE CREATE documents DIM 384
+
+# Create dataset with schema
+LANCE CREATE products DIM 768 SCHEMA category:string price:float available:bool
+
+# List datasets
+LANCE LIST
+
+# Get dataset information
+LANCE INFO documents
+```
+
+#### Data Operations
+```bash
+# Store text with metadata
+LANCE STORE documents TEXT "Machine learning tutorial" category "education" author "John Doe"
+
+# Store image with metadata
+LANCE STORE images IMAGE "base64_encoded_image..." filename "photo.jpg" tags "nature,landscape"
+
+# Store multimodal content
+LANCE STORE content TEXT "Product description" IMAGE "base64_image..." type "product"
+```
+
+#### Search Operations
+```bash
+# Search with raw vector
+LANCE SEARCH documents VECTOR "0.1,0.2,0.3,0.4" K 5
+
+# Semantic text search
+LANCE SEARCH.TEXT documents "artificial intelligence" K 10 NPROBES 20
+
+# Generate embeddings
+LANCE EMBED.TEXT "Hello world" "Machine learning"
+```
+
+#### Index Management
+```bash
+# Create performance index
+LANCE CREATE.INDEX documents IVF_PQ PARTITIONS 256 SUBVECTORS 16
+
+# Drop dataset
+LANCE DROP old_dataset
+```
+
+### Configuration
+
+#### Setting Up Embedding Service
+```bash
+# Configure embedding service URL
+redis-cli HSET config:core:aiembed url "http://your-embedding-service:8080/embed"
+
+# Optional: Set authentication token
+redis-cli HSET config:core:aiembed token "your-api-token"
+```
+
+#### Embedding Service API
+Your embedding service should accept POST requests:
+```json
+{
+  "texts": ["text1", "text2"],
+  "images": ["base64_image1", "base64_image2"],
+  "model": "your-model-name"
+}
+```
+
+And return responses:
+```json
+{
+  "embeddings": [[0.1, 0.2, ...], [0.3, 0.4, ...]],
+  "model": "model-name",
+  "usage": {"tokens": 100, "requests": 2}
+}
+```
+
+### Interactive Features
+
+The demo script includes:
+- **Colored output** for better readability
+- **Step-by-step execution** with explanations
+- **Error handling** demonstrations
+- **Automatic cleanup** options
+- **Performance testing** concepts
+- **Real-world usage** examples
+
+### Use Cases Demonstrated
+
+1. **Document Search System**
+   - Semantic document retrieval
+   - Metadata filtering
+   - Relevance ranking
+
+2. **Image Similarity Search**
+   - Visual content matching
+   - Tag-based filtering
+   - Multimodal queries
+
+3. **Product Recommendations**
+   - Feature-based similarity
+   - Category filtering
+   - Price range queries
+
+4. **Content Management**
+   - Mixed media storage
+   - Cross-modal search
+   - Rich metadata support
+
+---
 
 ## Tantivy Search Demo (Bash Script)
 

examples/lance_vector_demo.sh

@@ -0,0 +1,426 @@
+#!/bin/bash
+
+# Lance Vector Database Demo Script
+# This script demonstrates all Lance vector database operations in HeroDB
+
+set -e  # Exit on any error
+
+# Configuration
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_CLI="redis-cli -h $REDIS_HOST -p $REDIS_PORT"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Helper functions
+log_info() {
+    echo -e "${BLUE}[INFO]${NC} $1"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+execute_command() {
+    local cmd="$1"
+    local description="$2"
+    
+    echo
+    log_info "Executing: $description"
+    echo "Command: $cmd"
+    
+    if result=$($cmd 2>&1); then
+        log_success "Result: $result"
+    else
+        log_error "Failed: $result"
+        return 1
+    fi
+}
+
+# Check if HeroDB is running
+check_herodb() {
+    log_info "Checking if HeroDB is running..."
+    if ! $REDIS_CLI ping > /dev/null 2>&1; then
+        log_error "HeroDB is not running. Please start it first:"
+        echo "  cargo run -- --dir ./test_data --port $REDIS_PORT"
+        exit 1
+    fi
+    log_success "HeroDB is running"
+}
+
+# Setup embedding service configuration
+setup_embedding_service() {
+    log_info "Setting up embedding service configuration..."
+    
+    # Note: This is a mock URL for demonstration
+    # In production, replace with your actual embedding service
+    execute_command \
+        "$REDIS_CLI HSET config:core:aiembed url 'http://localhost:8080/embed'" \
+        "Configure embedding service URL"
+    
+    # Optional: Set authentication token
+    # execute_command \
+    #     "$REDIS_CLI HSET config:core:aiembed token 'your-api-token'" \
+    #     "Configure embedding service token"
+    
+    log_warning "Note: Embedding service at http://localhost:8080/embed is not running."
+    log_warning "Some operations will fail, but this demonstrates the command structure."
+}
+
+# Dataset Management Operations
+demo_dataset_management() {
+    echo
+    echo "=========================================="
+    echo "         DATASET MANAGEMENT DEMO"
+    echo "=========================================="
+    
+    # List datasets (should be empty initially)
+    execute_command \
+        "$REDIS_CLI LANCE LIST" \
+        "List all datasets (initially empty)"
+    
+    # Create a simple dataset
+    execute_command \
+        "$REDIS_CLI LANCE CREATE documents DIM 384" \
+        "Create a simple document dataset with 384 dimensions"
+    
+    # Create a dataset with schema
+    execute_command \
+        "$REDIS_CLI LANCE CREATE products DIM 768 SCHEMA category:string price:float available:bool description:string" \
+        "Create products dataset with custom schema"
+    
+    # Create an image dataset
+    execute_command \
+        "$REDIS_CLI LANCE CREATE images DIM 512 SCHEMA filename:string tags:string width:int height:int" \
+        "Create images dataset for multimodal content"
+    
+    # List datasets again
+    execute_command \
+        "$REDIS_CLI LANCE LIST" \
+        "List all datasets (should show 3 datasets)"
+    
+    # Get info about datasets
+    execute_command \
+        "$REDIS_CLI LANCE INFO documents" \
+        "Get information about documents dataset"
+    
+    execute_command \
+        "$REDIS_CLI LANCE INFO products" \
+        "Get information about products dataset"
+}
+
+# Embedding Operations
+demo_embedding_operations() {
+    echo
+    echo "=========================================="
+    echo "         EMBEDDING OPERATIONS DEMO"
+    echo "=========================================="
+    
+    log_warning "The following operations will fail because no embedding service is running."
+    log_warning "This demonstrates the command structure and error handling."
+    
+    # Try to embed text (will fail without embedding service)
+    execute_command \
+        "$REDIS_CLI LANCE EMBED.TEXT 'Hello world'" \
+        "Generate embedding for single text" || true
+    
+    # Try to embed multiple texts
+    execute_command \
+        "$REDIS_CLI LANCE EMBED.TEXT 'Machine learning' 'Artificial intelligence' 'Deep learning'" \
+        "Generate embeddings for multiple texts" || true
+}
+
+# Data Storage Operations
+demo_data_storage() {
+    echo
+    echo "=========================================="
+    echo "         DATA STORAGE DEMO"
+    echo "=========================================="
+    
+    log_warning "Storage operations will fail without embedding service, but show command structure."
+    
+    # Store text documents
+    execute_command \
+        "$REDIS_CLI LANCE STORE documents TEXT 'Introduction to machine learning algorithms and their applications in modern AI systems' category 'education' author 'John Doe' difficulty 'beginner'" \
+        "Store a document with text and metadata" || true
+    
+    execute_command \
+        "$REDIS_CLI LANCE STORE documents TEXT 'Deep learning neural networks for computer vision tasks' category 'research' author 'Jane Smith' difficulty 'advanced'" \
+        "Store another document" || true
+    
+    # Store product information
+    execute_command \
+        "$REDIS_CLI LANCE STORE products TEXT 'High-performance laptop with 16GB RAM and SSD storage' category 'electronics' price '1299.99' available 'true'" \
+        "Store product with text description" || true
+    
+    # Store image with metadata (using placeholder base64)
+    execute_command \
+        "$REDIS_CLI LANCE STORE images IMAGE 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==' filename 'sample.png' tags 'test,demo' width '1' height '1'" \
+        "Store image with metadata (1x1 pixel PNG)" || true
+    
+    # Store multimodal content
+    execute_command \
+        "$REDIS_CLI LANCE STORE images TEXT 'Beautiful sunset over mountains' IMAGE 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==' filename 'sunset.png' tags 'nature,landscape' location 'California'" \
+        "Store multimodal content (text + image)" || true
+}
+
+# Search Operations
+demo_search_operations() {
+    echo
+    echo "=========================================="
+    echo "         SEARCH OPERATIONS DEMO"
+    echo "=========================================="
+    
+    log_warning "Search operations will fail without data, but show command structure."
+    
+    # Search with raw vector
+    execute_command \
+        "$REDIS_CLI LANCE SEARCH documents VECTOR '0.1,0.2,0.3,0.4,0.5' K 5" \
+        "Search with raw vector (5 results)" || true
+    
+    # Search with vector and parameters
+    execute_command \
+        "$REDIS_CLI LANCE SEARCH documents VECTOR '0.1,0.2,0.3,0.4,0.5' K 10 NPROBES 20 REFINE 2" \
+        "Search with vector and advanced parameters" || true
+    
+    # Text-based search
+    execute_command \
+        "$REDIS_CLI LANCE SEARCH.TEXT documents 'machine learning algorithms' K 5" \
+        "Search using text query" || true
+    
+    # Text search with parameters
+    execute_command \
+        "$REDIS_CLI LANCE SEARCH.TEXT products 'laptop computer' K 3 NPROBES 10" \
+        "Search products using text with parameters" || true
+    
+    # Search in image dataset
+    execute_command \
+        "$REDIS_CLI LANCE SEARCH.TEXT images 'sunset landscape' K 5" \
+        "Search images using text description" || true
+}
+
+# Index Management Operations
+demo_index_management() {
+    echo
+    echo "=========================================="
+    echo "         INDEX MANAGEMENT DEMO"
+    echo "=========================================="
+    
+    # Create indexes for better search performance
+    execute_command \
+        "$REDIS_CLI LANCE CREATE.INDEX documents IVF_PQ" \
+        "Create default IVF_PQ index for documents"
+    
+    execute_command \
+        "$REDIS_CLI LANCE CREATE.INDEX products IVF_PQ PARTITIONS 512 SUBVECTORS 32" \
+        "Create IVF_PQ index with custom parameters for products"
+    
+    execute_command \
+        "$REDIS_CLI LANCE CREATE.INDEX images IVF_PQ PARTITIONS 256 SUBVECTORS 16" \
+        "Create IVF_PQ index for images dataset"
+    
+    log_success "Indexes created successfully"
+}
+
+# Advanced Usage Examples
+demo_advanced_usage() {
+    echo
+    echo "=========================================="
+    echo "         ADVANCED USAGE EXAMPLES"
+    echo "=========================================="
+    
+    # Create a specialized dataset for semantic search
+    execute_command \
+        "$REDIS_CLI LANCE CREATE semantic_search DIM 1536 SCHEMA title:string content:string url:string timestamp:string source:string" \
+        "Create dataset for semantic search with rich metadata"
+    
+    # Demonstrate batch operations concept
+    log_info "Batch operations example (would store multiple items):"
+    echo "  for doc in documents:"
+    echo "    LANCE STORE semantic_search TEXT \"\$doc_content\" title \"\$title\" url \"\$url\""
+    
+    # Show monitoring commands
+    log_info "Monitoring and maintenance commands:"
+    execute_command \
+        "$REDIS_CLI LANCE LIST" \
+        "List all datasets for monitoring"
+    
+    # Show dataset statistics
+    for dataset in documents products images semantic_search; do
+        execute_command \
+            "$REDIS_CLI LANCE INFO $dataset" \
+            "Get statistics for $dataset" || true
+    done
+}
+
+# Cleanup Operations
+demo_cleanup() {
+    echo
+    echo "=========================================="
+    echo "         CLEANUP OPERATIONS DEMO"
+    echo "=========================================="
+    
+    log_info "Demonstrating cleanup operations..."
+    
+    # Drop individual datasets
+    execute_command \
+        "$REDIS_CLI LANCE DROP semantic_search" \
+        "Drop semantic_search dataset"
+    
+    # List remaining datasets
+    execute_command \
+        "$REDIS_CLI LANCE LIST" \
+        "List remaining datasets"
+    
+    # Ask user if they want to clean up all test data
+    echo
+    read -p "Do you want to clean up all test datasets? (y/N): " -n 1 -r
+    echo
+    if [[ $REPLY =~ ^[Yy]$ ]]; then
+        execute_command \
+            "$REDIS_CLI LANCE DROP documents" \
+            "Drop documents dataset"
+        
+        execute_command \
+            "$REDIS_CLI LANCE DROP products" \
+            "Drop products dataset"
+        
+        execute_command \
+            "$REDIS_CLI LANCE DROP images" \
+            "Drop images dataset"
+        
+        execute_command \
+            "$REDIS_CLI LANCE LIST" \
+            "Verify all datasets are cleaned up"
+        
+        log_success "All test datasets cleaned up"
+    else
+        log_info "Keeping test datasets for further experimentation"
+    fi
+}
+
+# Error Handling Demo
+demo_error_handling() {
+    echo
+    echo "=========================================="
+    echo "         ERROR HANDLING DEMO"
+    echo "=========================================="
+    
+    log_info "Demonstrating various error conditions..."
+    
+    # Try to access non-existent dataset
+    execute_command \
+        "$REDIS_CLI LANCE INFO nonexistent_dataset" \
+        "Try to get info for non-existent dataset" || true
+    
+    # Try to search non-existent dataset
+    execute_command \
+        "$REDIS_CLI LANCE SEARCH nonexistent_dataset VECTOR '0.1,0.2' K 5" \
+        "Try to search non-existent dataset" || true
+    
+    # Try to drop non-existent dataset
+    execute_command \
+        "$REDIS_CLI LANCE DROP nonexistent_dataset" \
+        "Try to drop non-existent dataset" || true
+    
+    # Try invalid vector format
+    execute_command \
+        "$REDIS_CLI LANCE SEARCH documents VECTOR 'invalid,vector,format' K 5" \
+        "Try search with invalid vector format" || true
+    
+    log_info "Error handling demonstration complete"
+}
+
+# Performance Testing Demo
+demo_performance_testing() {
+    echo
+    echo "=========================================="
+    echo "         PERFORMANCE TESTING DEMO"
+    echo "=========================================="
+    
+    log_info "Creating performance test dataset..."
+    execute_command \
+        "$REDIS_CLI LANCE CREATE perf_test DIM 128 SCHEMA batch_id:string item_id:string" \
+        "Create performance test dataset"
+    
+    log_info "Performance testing would involve:"
+    echo "  1. Bulk loading thousands of vectors"
+    echo "  2. Creating indexes with different parameters"
+    echo "  3. Measuring search latency with various K values"
+    echo "  4. Testing different NPROBES settings"
+    echo "  5. Monitoring memory usage"
+    
+    log_info "Example performance test commands:"
+    echo "  # Test search speed with different parameters"
+    echo "  time redis-cli LANCE SEARCH.TEXT perf_test 'query' K 10"
+    echo "  time redis-cli LANCE SEARCH.TEXT perf_test 'query' K 10 NPROBES 50"
+    echo "  time redis-cli LANCE SEARCH.TEXT perf_test 'query' K 100 NPROBES 100"
+    
+    # Clean up performance test dataset
+    execute_command \
+        "$REDIS_CLI LANCE DROP perf_test" \
+        "Clean up performance test dataset"
+}
+
+# Main execution
+main() {
+    echo "=========================================="
+    echo "    LANCE VECTOR DATABASE DEMO SCRIPT"
+    echo "=========================================="
+    echo
+    echo "This script demonstrates all Lance vector database operations."
+    echo "Note: Some operations will fail without a running embedding service."
+    echo "This is expected and demonstrates error handling."
+    echo
+    
+    # Check prerequisites
+    check_herodb
+    
+    # Setup
+    setup_embedding_service
+    
+    # Run demos
+    demo_dataset_management
+    demo_embedding_operations
+    demo_data_storage
+    demo_search_operations
+    demo_index_management
+    demo_advanced_usage
+    demo_error_handling
+    demo_performance_testing
+    
+    # Cleanup
+    demo_cleanup
+    
+    echo
+    echo "=========================================="
+    echo "         DEMO COMPLETE"
+    echo "=========================================="
+    echo
+    log_success "Lance vector database demo completed successfully!"
+    echo
+    echo "Next steps:"
+    echo "1. Set up a real embedding service (OpenAI, Hugging Face, etc.)"
+    echo "2. Update the embedding service URL configuration"
+    echo "3. Try storing and searching real data"
+    echo "4. Experiment with different vector dimensions and index parameters"
+    echo "5. Build your AI-powered application!"
+    echo
+    echo "For more information, see docs/lance_vector_db.md"
+}
+
+# Run the demo
+main "$@"

src/admin_meta.rs

@@ -1,501 +0,0 @@
-use std::path::{Path, PathBuf};
-use std::sync::{Arc, OnceLock, Mutex, RwLock};
-use std::collections::HashMap;
-
-use crate::error::DBError;
-use crate::options;
-use crate::rpc::Permissions;
-use crate::storage::Storage;
-use crate::storage_sled::SledStorage;
-use crate::storage_trait::StorageBackend;
-
-// Key builders
-fn k_admin_next_id() -> &'static str {
-    "admin:next_id"
-}
-fn k_admin_dbs() -> &'static str {
-    "admin:dbs"
-}
-fn k_meta_db(id: u64) -> String {
-    format!("meta:db:{}", id)
-}
-fn k_meta_db_keys(id: u64) -> String {
-    format!("meta:db:{}:keys", id)
-}
-fn k_meta_db_enc(id: u64) -> String {
-    format!("meta:db:{}:enc", id)
-}
-
-// Global cache of admin DB 0 handles per base_dir to avoid sled/reDB file-lock contention
-// and to correctly isolate different test instances with distinct directories.
-static ADMIN_STORAGES: OnceLock<RwLock<HashMap<String, Arc<dyn StorageBackend>>>> = OnceLock::new();
-
-// Global registry for data DB storages to avoid double-open across process.
-static DATA_STORAGES: OnceLock<RwLock<HashMap<u64, Arc<dyn StorageBackend>>>> = OnceLock::new();
-static DATA_INIT_LOCK: Mutex<()> = Mutex::new(());
-
-fn init_admin_storage(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-) -> Result<Arc<dyn StorageBackend>, DBError> {
-    let db_file = base_dir.join("0.db");
-    if let Some(parent_dir) = db_file.parent() {
-        std::fs::create_dir_all(parent_dir).map_err(|e| {
-            DBError(format!("Failed to create directory {}: {}", parent_dir.display(), e))
-        })?;
-    }
-    let storage: Arc<dyn StorageBackend> = match backend {
-        options::BackendType::Redb => Arc::new(Storage::new(&db_file, true, Some(admin_secret))?),
-        options::BackendType::Sled => Arc::new(SledStorage::new(&db_file, true, Some(admin_secret))?),
-        options::BackendType::Tantivy | options::BackendType::Lance => {
-            return Err(DBError("Admin DB 0 cannot use search-only backends (Tantivy/Lance)".to_string()))
-        }
-    };
-    Ok(storage)
-}
-
-// Get or initialize a cached handle to admin DB 0 per base_dir (thread-safe, no double-open race)
-pub fn open_admin_storage(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-) -> Result<Arc<dyn StorageBackend>, DBError> {
-    let map = ADMIN_STORAGES.get_or_init(|| RwLock::new(HashMap::new()));
-    let key = base_dir.display().to_string();
-    // Fast path
-    if let Some(st) = map.read().unwrap().get(&key) {
-        return Ok(st.clone());
-    }
-    // Slow path with write lock
-    {
-        let mut w = map.write().unwrap();
-        if let Some(st) = w.get(&key) {
-            return Ok(st.clone());
-        }
-
-        // Detect existing 0.db backend by filesystem, if present.
-        let admin_path = base_dir.join("0.db");
-        let detected = if admin_path.exists() {
-            if admin_path.is_file() {
-                Some(options::BackendType::Redb)
-            } else if admin_path.is_dir() {
-                Some(options::BackendType::Sled)
-            } else {
-                None
-            }
-        } else {
-            None
-        };
-
-        let effective_backend = match detected {
-            Some(d) if d != backend => {
-                eprintln!(
-                    "warning: Admin DB 0 at {} appears to be {:?}, but process default is {:?}. Using detected backend.",
-                    admin_path.display(),
-                    d,
-                    backend
-                );
-                d
-            }
-            Some(d) => d,
-            None => backend, // First boot: use requested backend to initialize 0.db
-        };
-
-        let st = init_admin_storage(base_dir, effective_backend, admin_secret)?;
-        w.insert(key, st.clone());
-        Ok(st)
-    }
-}
-
-// Ensure admin structures exist in encrypted DB 0
-pub fn ensure_bootstrap(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-) -> Result<(), DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-
-    // Initialize next id if missing
-    if !admin.exists(k_admin_next_id())? {
-        admin.set(k_admin_next_id().to_string(), "1".to_string())?;
-    }
-    // admin:dbs is a hash; it's fine if it doesn't exist (hlen -> 0)
-    Ok(())
-}
-
-// Get or initialize a shared handle to a data DB (> 0), avoiding double-open across subsystems
-pub fn open_data_storage(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-) -> Result<Arc<dyn StorageBackend>, DBError> {
-    if id == 0 {
-        return open_admin_storage(base_dir, backend, admin_secret);
-    }
-
-    // Validate existence in admin metadata
-    if !db_exists(base_dir, backend.clone(), admin_secret, id)? {
-        return Err(DBError(format!(
-            "Cannot open database instance {}, as that database instance does not exist.",
-            id
-        )));
-    }
-
-    let map = DATA_STORAGES.get_or_init(|| RwLock::new(HashMap::new()));
-    // Fast path
-    if let Some(st) = map.read().unwrap().get(&id) {
-        return Ok(st.clone());
-    }
-
-    // Slow path with init lock
-    let _guard = DATA_INIT_LOCK.lock().unwrap();
-    if let Some(st) = map.read().unwrap().get(&id) {
-        return Ok(st.clone());
-    }
-
-    // Resolve effective backend for this db id:
-    // 1) Try admin meta "backend" field
-    // 2) If missing, sniff filesystem (file => Redb, dir => Sled), then persist into admin meta
-    // 3) Fallback to requested 'backend' (startup default) if nothing else is known
-    let meta_backend = get_database_backend(base_dir, backend.clone(), admin_secret, id).ok().flatten();
-    let db_path = base_dir.join(format!("{}.db", id));
-    let sniffed_backend = if db_path.exists() {
-        if db_path.is_file() {
-            Some(options::BackendType::Redb)
-        } else if db_path.is_dir() {
-            Some(options::BackendType::Sled)
-        } else {
-            None
-        }
-    } else {
-        None
-    };
-    let effective_backend = meta_backend.clone().or(sniffed_backend).unwrap_or(backend.clone());
-
-    // If we had to sniff (i.e., meta missing), persist it for future robustness
-    if meta_backend.is_none() {
-        let _ = set_database_backend(base_dir, backend.clone(), admin_secret, id, effective_backend.clone());
-    }
-
-    // Warn if caller-provided backend differs from effective
-    if effective_backend != backend {
-        eprintln!(
-            "notice: Database {} backend resolved to {:?} (caller requested {:?}). Using resolved backend.",
-            id, effective_backend, backend
-        );
-    }
-
-    // Determine per-db encryption (from admin meta)
-    let enc = get_enc_key(base_dir, backend.clone(), admin_secret, id)?;
-    let should_encrypt = enc.is_some();
-
-    // Build database file path and ensure parent dir exists
-    let db_file = PathBuf::from(base_dir).join(format!("{}.db", id));
-    if let Some(parent_dir) = db_file.parent() {
-        std::fs::create_dir_all(parent_dir).map_err(|e| {
-            DBError(format!("Failed to create directory {}: {}", parent_dir.display(), e))
-        })?;
-    }
-
-    // Open storage using the effective backend
-    let storage: Arc<dyn StorageBackend> = match effective_backend {
-        options::BackendType::Redb => Arc::new(Storage::new(&db_file, should_encrypt, enc.as_deref())?),
-        options::BackendType::Sled => Arc::new(SledStorage::new(&db_file, should_encrypt, enc.as_deref())?),
-        options::BackendType::Tantivy => {
-            return Err(DBError("Tantivy backend has no KV storage; use FT.* commands only".to_string()))
-        }
-        options::BackendType::Lance => {
-            return Err(DBError("Lance backend has no KV storage; use LANCE.* commands only".to_string()))
-        }
-    };
-
-    // Publish to registry
-    map.write().unwrap().insert(id, storage.clone());
-    Ok(storage)
-}
-
-// Allocate the next DB id and persist new pointer
-pub fn allocate_next_id(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-) -> Result<u64, DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let cur = admin
-        .get(k_admin_next_id())?
-        .unwrap_or_else(|| "1".to_string());
-    let id: u64 = cur.parse().unwrap_or(1);
-    let next = id.checked_add(1).ok_or_else(|| DBError("next_id overflow".into()))?;
-    admin.set(k_admin_next_id().to_string(), next.to_string())?;
-
-    // Register into admin:dbs set/hash
-    let _ = admin.hset(k_admin_dbs(), vec![(id.to_string(), "1".to_string())])?;
-
-    // Default meta for the new db: public true
-    let meta_key = k_meta_db(id);
-    let _ = admin.hset(&meta_key, vec![("public".to_string(), "true".to_string())])?;
-
-    Ok(id)
-}
-
-// Check existence of a db id in admin:dbs
-pub fn db_exists(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-) -> Result<bool, DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    Ok(admin.hexists(k_admin_dbs(), &id.to_string())?)
-}
-
-// Get per-db encryption key, if any
-pub fn get_enc_key(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-) -> Result<Option<String>, DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    admin.get(&k_meta_db_enc(id))
-}
-
-// Set per-db encryption key (called during create)
-pub fn set_enc_key(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-    key: &str,
-) -> Result<(), DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    admin.set(k_meta_db_enc(id), key.to_string())
-}
-
-// Set database public flag
-pub fn set_database_public(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-    public: bool,
-) -> Result<(), DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let mk = k_meta_db(id);
-    let _ = admin.hset(&mk, vec![("public".to_string(), public.to_string())])?;
-    Ok(())
-}
-
-// Persist per-db backend type in admin metadata (module-scope)
-pub fn set_database_backend(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-    db_backend: options::BackendType,
-) -> Result<(), DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let mk = k_meta_db(id);
-    let val = match db_backend {
-        options::BackendType::Redb => "Redb",
-        options::BackendType::Sled => "Sled",
-        options::BackendType::Tantivy => "Tantivy",
-        options::BackendType::Lance => "Lance",
-    };
-    let _ = admin.hset(&mk, vec![("backend".to_string(), val.to_string())])?;
-    Ok(())
-}
-
-pub fn get_database_backend(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-) -> Result<Option<options::BackendType>, DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let mk = k_meta_db(id);
-    match admin.hget(&mk, "backend")? {
-        Some(s) if s == "Redb" => Ok(Some(options::BackendType::Redb)),
-        Some(s) if s == "Sled" => Ok(Some(options::BackendType::Sled)),
-        Some(s) if s == "Tantivy" => Ok(Some(options::BackendType::Tantivy)),
-        Some(s) if s == "Lance" => Ok(Some(options::BackendType::Lance)),
-        _ => Ok(None),
-    }
-}
-
-// Set database name
-pub fn set_database_name(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-    name: &str,
-) -> Result<(), DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let mk = k_meta_db(id);
-    let _ = admin.hset(&mk, vec![("name".to_string(), name.to_string())])?;
-    Ok(())
-}
-
-// Get database name
-pub fn get_database_name(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-) -> Result<Option<String>, DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let mk = k_meta_db(id);
-    admin.hget(&mk, "name")
-}
-
-// Internal: load public flag; default to true when meta missing
-fn load_public(
-    admin: &Arc<dyn StorageBackend>,
-    id: u64,
-) -> Result<bool, DBError> {
-    let mk = k_meta_db(id);
-    match admin.hget(&mk, "public")? {
-        Some(v) => Ok(v == "true"),
-        None => Ok(true),
-    }
-}
-
-// Add access key for db (value format: "Read:ts" or "ReadWrite:ts")
-pub fn add_access_key(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-    key_plain: &str,
-    perms: Permissions,
-) -> Result<(), DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let hash = crate::rpc::hash_key(key_plain);
-    let v = match perms {
-        Permissions::Read => format!("Read:{}", now_secs()),
-        Permissions::ReadWrite => format!("ReadWrite:{}", now_secs()),
-    };
-    let _ = admin.hset(&k_meta_db_keys(id), vec![(hash, v)])?;
-    Ok(())
-}
-
-// Delete access key by hash
-pub fn delete_access_key(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-    key_hash: &str,
-) -> Result<bool, DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let n = admin.hdel(&k_meta_db_keys(id), vec![key_hash.to_string()])?;
-    Ok(n > 0)
-}
-
-// List access keys, returning (hash, perms, created_at_secs)
-pub fn list_access_keys(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-) -> Result<Vec<(String, Permissions, u64)>, DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let pairs = admin.hgetall(&k_meta_db_keys(id))?;
-    let mut out = Vec::new();
-    for (hash, val) in pairs {
-        let (perm, ts) = parse_perm_value(&val);
-        out.push((hash, perm, ts));
-    }
-    Ok(out)
-}
-
-// Verify access permission for db id with optional key
-// Returns:
-// - Ok(Some(Permissions)) when access is allowed
-// - Ok(None) when not allowed or db missing (caller can distinguish by calling db_exists)
-pub fn verify_access(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-    id: u64,
-    key_opt: Option<&str>,
-) -> Result<Option<Permissions>, DBError> {
-    // Admin DB 0: require exact admin_secret
-    if id == 0 {
-        if let Some(k) = key_opt {
-            if k == admin_secret {
-                return Ok(Some(Permissions::ReadWrite));
-            }
-        }
-        return Ok(None);
-    }
-
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    if !admin.hexists(k_admin_dbs(), &id.to_string())? {
-        return Ok(None);
-    }
-
-    let is_public = load_public(&admin, id)?;
-
-    // If a key is explicitly provided, enforce its validity strictly.
-    // Do NOT fall back to public when an invalid key is supplied.
-    if let Some(k) = key_opt {
-        let hash = crate::rpc::hash_key(k);
-        if let Some(v) = admin.hget(&k_meta_db_keys(id), &hash)? {
-            let (perm, _ts) = parse_perm_value(&v);
-            return Ok(Some(perm));
-        }
-        // Invalid key
-        return Ok(None);
-    }
-
-    // No key provided: allow access if DB is public, otherwise deny
-    if is_public {
-        Ok(Some(Permissions::ReadWrite))
-    } else {
-        Ok(None)
-    }
-}
-
-// Enumerate all db ids
-pub fn list_dbs(
-    base_dir: &Path,
-    backend: options::BackendType,
-    admin_secret: &str,
-) -> Result<Vec<u64>, DBError> {
-    let admin = open_admin_storage(base_dir, backend, admin_secret)?;
-    let ids = admin.hkeys(k_admin_dbs())?;
-    let mut out = Vec::new();
-    for s in ids {
-        if let Ok(v) = s.parse() {
-            out.push(v);
-        }
-    }
-    Ok(out)
-}
-
-// Helper: parse permission value "Read:ts" or "ReadWrite:ts"
-fn parse_perm_value(v: &str) -> (Permissions, u64) {
-    let mut parts = v.split(':');
-    let p = parts.next().unwrap_or("Read");
-    let ts = parts
-        .next()
-        .and_then(|s| s.parse().ok())
-        .unwrap_or(0u64);
-    let perm = match p {
-        "ReadWrite" => Permissions::ReadWrite,
-        _ => Permissions::Read,
-    };
-    (perm, ts)
-}
-
-fn now_secs() -> u64 {
-    use std::time::{SystemTime, UNIX_EPOCH};
-    SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .unwrap_or_default()
-        .as_secs()
-}

src/age.rs

@@ -19,8 +19,6 @@ use age::x25519;
 use ed25519_dalek::{Signature, Signer, Verifier, SigningKey, VerifyingKey};
 
 use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
-use std::collections::HashSet;
-use std::convert::TryInto;
 
 use crate::protocol::Protocol;
 use crate::server::Server;
@@ -76,125 +74,6 @@ fn parse_ed25519_verifying_key(s: &str) -> Result<VerifyingKey, AgeWireError> {
     VerifyingKey::from_bytes(&key_bytes).map_err(|_| AgeWireError::ParseKey)
 }
 
-// ---------- Derivation + Raw X25519 (Ed25519 -> X25519) ----------
-//
-// We deterministically derive an X25519 keypair from an Ed25519 SigningKey.
-// We persist the X25519 public/secret as base64-encoded 32-byte raw values
-// (no "age1..."/"AGE-SECRET-KEY-1..." formatting). Name-based encrypt/decrypt
-// uses these raw values directly via x25519-dalek + ChaCha20Poly1305.
-
-use chacha20poly1305::{aead::{Aead, KeyInit}, ChaCha20Poly1305, Key, Nonce};
-use sha2::{Digest, Sha256};
-use x25519_dalek::{PublicKey as XPublicKey, StaticSecret as XStaticSecret};
-
-fn derive_x25519_raw_from_ed25519(sk: &SigningKey) -> ([u8; 32], [u8; 32]) {
-    // X25519 secret scalar (clamped) from Ed25519 secret
-    let scalar: [u8; 32] = sk.to_scalar_bytes();
-    // Build X25519 secret/public using dalek
-    let xsec = XStaticSecret::from(scalar);
-    let xpub = XPublicKey::from(&xsec);
-    (xpub.to_bytes(), xsec.to_bytes())
-}
-
-fn derive_x25519_raw_b64_from_ed25519(sk: &SigningKey) -> (String, String) {
-    let (xpub, xsec) = derive_x25519_raw_from_ed25519(sk);
-    (B64.encode(xpub), B64.encode(xsec))
-}
-
-// Helper: detect whether a stored key looks like an age-formatted string
-fn looks_like_age_format(s: &str) -> bool {
-    s.starts_with("age1") || s.starts_with("AGE-SECRET-KEY-1")
-}
-
-// Our container format for name-based raw X25519 encryption:
-// bytes = "HDBX1" (5) || eph_pub(32) || nonce(12) || ciphertext(..)
-// Entire blob is base64-encoded for transport.
-const HDBX1_MAGIC: &[u8; 5] = b"HDBX1";
-
-fn encrypt_b64_with_x25519_raw(recip_pub_b64: &str, msg: &str) -> Result<String, AgeWireError> {
-    use rand::RngCore;
-    use rand::rngs::OsRng;
-
-    // Parse recipient public key (raw 32 bytes, base64)
-    let recip_pub_bytes = B64.decode(recip_pub_b64).map_err(|_| AgeWireError::ParseKey)?;
-    if recip_pub_bytes.len() != 32 { return Err(AgeWireError::ParseKey); }
-    let recip_pub_arr: [u8; 32] = recip_pub_bytes.as_slice().try_into().map_err(|_| AgeWireError::ParseKey)?;
-    let recip_pub: XPublicKey = XPublicKey::from(recip_pub_arr);
-
-    // Generate ephemeral X25519 keypair
-    let mut eph_sec_bytes = [0u8; 32];
-    OsRng.fill_bytes(&mut eph_sec_bytes);
-    let eph_sec = XStaticSecret::from(eph_sec_bytes);
-    let eph_pub = XPublicKey::from(&eph_sec);
-
-    // ECDH
-    let shared = eph_sec.diffie_hellman(&recip_pub);
-    // Derive symmetric key via SHA-256 over context + shared + parties
-    let mut hasher = Sha256::default();
-    hasher.update(b"herodb-x25519-v1");
-    hasher.update(shared.as_bytes());
-    hasher.update(eph_pub.as_bytes());
-    hasher.update(recip_pub.as_bytes());
-    let key_bytes = hasher.finalize();
-    let key = Key::from_slice(&key_bytes[..32]);
-
-    // Nonce (12 bytes)
-    let mut nonce_bytes = [0u8; 12];
-    OsRng.fill_bytes(&mut nonce_bytes);
-    let nonce = Nonce::from_slice(&nonce_bytes);
-
-    // Encrypt
-    let cipher = ChaCha20Poly1305::new(key);
-    let ct = cipher.encrypt(nonce, msg.as_bytes())
-        .map_err(|e| AgeWireError::Crypto(format!("encrypt: {e}")))?;
-
-    // Assemble container
-    let mut out = Vec::with_capacity(5 + 32 + 12 + ct.len());
-    out.extend_from_slice(HDBX1_MAGIC);
-    out.extend_from_slice(eph_pub.as_bytes());
-    out.extend_from_slice(&nonce_bytes);
-    out.extend_from_slice(&ct);
-
-    Ok(B64.encode(out))
-}
-
-fn decrypt_b64_with_x25519_raw(identity_sec_b64: &str, ct_b64: &str) -> Result<String, AgeWireError> {
-    // Parse X25519 secret (raw 32 bytes, base64)
-    let sec_bytes = B64.decode(identity_sec_b64).map_err(|_| AgeWireError::ParseKey)?;
-    if sec_bytes.len() != 32 { return Err(AgeWireError::ParseKey); }
-    let sec_arr: [u8; 32] = sec_bytes.as_slice().try_into().map_err(|_| AgeWireError::ParseKey)?;
-    let xsec = XStaticSecret::from(sec_arr);
-    let xpub = XPublicKey::from(&xsec); // self public
-
-    // Decode container
-    let blob = B64.decode(ct_b64.as_bytes()).map_err(|e| AgeWireError::Crypto(e.to_string()))?;
-    if blob.len() < 5 + 32 + 12 { return Err(AgeWireError::Crypto("ciphertext too short".to_string())); }
-    if &blob[..5] != HDBX1_MAGIC { return Err(AgeWireError::Crypto("bad header".to_string())); }
-
-    let eph_pub_arr: [u8; 32] = blob[5..5+32].try_into().map_err(|_| AgeWireError::Crypto("bad eph pub".to_string()))?;
-    let eph_pub = XPublicKey::from(eph_pub_arr);
-    let nonce_bytes: [u8; 12] = blob[5+32..5+32+12].try_into().unwrap();
-    let ct = &blob[5+32+12..];
-
-    // Recompute shared + key
-    let shared = xsec.diffie_hellman(&eph_pub);
-    let mut hasher = Sha256::default();
-    hasher.update(b"herodb-x25519-v1");
-    hasher.update(shared.as_bytes());
-    hasher.update(eph_pub.as_bytes());
-    hasher.update(xpub.as_bytes());
-    let key_bytes = hasher.finalize();
-    let key = Key::from_slice(&key_bytes[..32]);
-
-    // Decrypt
-    let cipher = ChaCha20Poly1305::new(key);
-    let nonce = Nonce::from_slice(&nonce_bytes);
-    let pt = cipher.decrypt(nonce, ct)
-        .map_err(|e| AgeWireError::Crypto(format!("decrypt: {e}")))?;
-
-    String::from_utf8(pt).map_err(|_| AgeWireError::Utf8)
-}
-
 // ---------- Stateless crypto helpers (string in/out) ----------
 
 pub fn gen_enc_keypair() -> (String, String) {
@@ -331,72 +210,13 @@ pub async fn cmd_age_verify(verify_pub: &str, message: &str, sig_b64: &str) -> P
     }
 }
 
-// ---------- NEW: unified stateless generator (Ed25519 + derived X25519 raw) ----------
-//
-// Returns 4-tuple:
-// [ verify_pub_b64 (32B), signpriv_b64 (32B), x25519_pub_b64 (32B), x25519_sec_b64 (32B) ]
-// No persistence (stateless).
-pub async fn cmd_age_genkey() -> Protocol {
-    use rand::RngCore;
-    use rand::rngs::OsRng;
-
-    let mut secret_bytes = [0u8; 32];
-    OsRng.fill_bytes(&mut secret_bytes);
-
-    let signing_key = SigningKey::from_bytes(&secret_bytes);
-    let verifying_key = signing_key.verifying_key();
-
-    let verify_b64 = B64.encode(verifying_key.to_bytes());
-    let sign_b64 = B64.encode(signing_key.to_bytes());
-
-    let (xpub_b64, xsec_b64) = derive_x25519_raw_b64_from_ed25519(&signing_key);
-
-    Protocol::Array(vec![
-        Protocol::BulkString(verify_b64),
-        Protocol::BulkString(sign_b64),
-        Protocol::BulkString(xpub_b64),
-        Protocol::BulkString(xsec_b64),
-    ])
-}
-
 // ---------- NEW: Persistent, named-key commands ----------
 
 pub async fn cmd_age_keygen(server: &Server, name: &str) -> Protocol {
-    use rand::RngCore;
-    use rand::rngs::OsRng;
-
-    // Generate Ed25519 keypair
-    let mut secret_bytes = [0u8; 32];
-    OsRng.fill_bytes(&mut secret_bytes);
-    let signing_key = SigningKey::from_bytes(&secret_bytes);
-    let verifying_key = signing_key.verifying_key();
-
-    // Encode Ed25519 as base64 (32 bytes)
-    let verify_b64 = B64.encode(verifying_key.to_bytes());
-    let sign_b64 = B64.encode(signing_key.to_bytes());
-
-    // Derive X25519 raw (32-byte) keys and encode as base64
-    let (xpub_b64, xsec_b64) = derive_x25519_raw_b64_from_ed25519(&signing_key);
-
-    // Decode to create age-formatted strings
-    let xpub_bytes = B64.decode(&xpub_b64).unwrap();
-    let xsec_bytes = B64.decode(&xsec_b64).unwrap();
-    let xpub_arr: [u8; 32] = xpub_bytes.as_slice().try_into().unwrap();
-    let xsec_arr: [u8; 32] = xsec_bytes.as_slice().try_into().unwrap();
-    let recip_str = format!("age1{}", B64.encode(xpub_arr));
-    let ident_str = format!("AGE-SECRET-KEY-1{}", B64.encode(xsec_arr));
-
-    // Persist Ed25519 and derived X25519 (key-managed mode)
-    if let Err(e) = sset(server, &sign_pub_key_key(name), &verify_b64) { return e.to_protocol(); }
-    if let Err(e) = sset(server, &sign_priv_key_key(name), &sign_b64) { return e.to_protocol(); }
-    if let Err(e) = sset(server, &enc_pub_key_key(name), &xpub_b64) { return e.to_protocol(); }
-    if let Err(e) = sset(server, &enc_priv_key_key(name), &xsec_b64) { return e.to_protocol(); }
-
-    // Return [recipient, identity] in age format
-    Protocol::Array(vec![
-        Protocol::BulkString(recip_str),
-        Protocol::BulkString(ident_str),
-    ])
+    let (recip, ident) = gen_enc_keypair();
+    if let Err(e) = sset(server, &enc_pub_key_key(name), &recip) { return e.to_protocol(); }
+    if let Err(e) = sset(server, &enc_priv_key_key(name), &ident) { return e.to_protocol(); }
+    Protocol::Array(vec![Protocol::BulkString(recip), Protocol::BulkString(ident)])
 }
 
 pub async fn cmd_age_signkeygen(server: &Server, name: &str) -> Protocol {
@@ -407,76 +227,26 @@ pub async fn cmd_age_signkeygen(server: &Server, name: &str) -> Protocol {
 }
 
 pub async fn cmd_age_encrypt_name(server: &Server, name: &str, message: &str) -> Protocol {
-    // Load stored recipient (could be raw b64 32-byte or "age1..." from legacy)
-    let recip_or_b64 = match sget(server, &enc_pub_key_key(name)) {
+    let recip = match sget(server, &enc_pub_key_key(name)) {
         Ok(Some(v)) => v,
-        Ok(None) => {
-            // Derive from stored Ed25519 if present, then persist
-            match sget(server, &sign_priv_key_key(name)) {
-                Ok(Some(sign_b64)) => {
-                    let sk = match parse_ed25519_signing_key(&sign_b64) {
-                        Ok(k) => k,
-                        Err(e) => return e.to_protocol(),
-                    };
-                    let (xpub_b64, xsec_b64) = derive_x25519_raw_b64_from_ed25519(&sk);
-                    if let Err(e) = sset(server, &enc_pub_key_key(name), &xpub_b64) { return e.to_protocol(); }
-                    if let Err(e) = sset(server, &enc_priv_key_key(name), &xsec_b64) { return e.to_protocol(); }
-                    xpub_b64
-                }
-                Ok(None) => return AgeWireError::NotFound("recipient (age:key:{name})").to_protocol(),
-                Err(e) => return e.to_protocol(),
-            }
-        }
+        Ok(None) => return AgeWireError::NotFound("recipient (age:key:{name})").to_protocol(),
         Err(e) => return e.to_protocol(),
     };
-
-    if looks_like_age_format(&recip_or_b64) {
-        match encrypt_b64(&recip_or_b64, message) {
-            Ok(ct) => Protocol::BulkString(ct),
-            Err(e) => e.to_protocol(),
-        }
-    } else {
-        match encrypt_b64_with_x25519_raw(&recip_or_b64, message) {
-            Ok(ct) => Protocol::BulkString(ct),
-            Err(e) => e.to_protocol(),
-        }
+    match encrypt_b64(&recip, message) {
+        Ok(ct) => Protocol::BulkString(ct),
+        Err(e) => e.to_protocol(),
     }
 }
 
 pub async fn cmd_age_decrypt_name(server: &Server, name: &str, ct_b64: &str) -> Protocol {
-    // Load stored identity (could be raw b64 32-byte or "AGE-SECRET-KEY-1..." from legacy)
-    let ident_or_b64 = match sget(server, &enc_priv_key_key(name)) {
+    let ident = match sget(server, &enc_priv_key_key(name)) {
         Ok(Some(v)) => v,
-        Ok(None) => {
-            // Derive from stored Ed25519 if present, then persist
-            match sget(server, &sign_priv_key_key(name)) {
-                Ok(Some(sign_b64)) => {
-                    let sk = match parse_ed25519_signing_key(&sign_b64) {
-                        Ok(k) => k,
-                        Err(e) => return e.to_protocol(),
-                    };
-                    let (xpub_b64, xsec_b64) = derive_x25519_raw_b64_from_ed25519(&sk);
-                    if let Err(e) = sset(server, &enc_pub_key_key(name), &xpub_b64) { return e.to_protocol(); }
-                    if let Err(e) = sset(server, &enc_priv_key_key(name), &xsec_b64) { return e.to_protocol(); }
-                    xsec_b64
-                }
-                Ok(None) => return AgeWireError::NotFound("identity (age:privkey:{name})").to_protocol(),
-                Err(e) => return e.to_protocol(),
-            }
-        }
+        Ok(None) => return AgeWireError::NotFound("identity (age:privkey:{name})").to_protocol(),
         Err(e) => return e.to_protocol(),
     };
-
-    if looks_like_age_format(&ident_or_b64) {
-        match decrypt_b64(&ident_or_b64, ct_b64) {
-            Ok(pt) => Protocol::BulkString(pt),
-            Err(e) => e.to_protocol(),
-        }
-    } else {
-        match decrypt_b64_with_x25519_raw(&ident_or_b64, ct_b64) {
-            Ok(pt) => Protocol::BulkString(pt),
-            Err(e) => e.to_protocol(),
-        }
+    match decrypt_b64(&ident, ct_b64) {
+        Ok(pt) => Protocol::BulkString(pt),
+        Err(e) => e.to_protocol(),
     }
 }
 
@@ -506,31 +276,33 @@ pub async fn cmd_age_verify_name(server: &Server, name: &str, message: &str, sig
 }
 
 pub async fn cmd_age_list(server: &Server) -> Protocol {
-    // Return a flat, deduplicated, sorted list of managed key names (no labels)
+    // Returns 4 arrays: ["encpub", <names...>], ["encpriv", ...], ["signpub", ...], ["signpriv", ...]
     let st = match server.current_storage() { Ok(s) => s, Err(e) => return Protocol::err(&e.0) };
 
     let pull = |pat: &str, prefix: &str| -> Result<Vec<String>, DBError> {
         let keys = st.keys(pat)?;
-        let mut names: Vec<String> = keys
-            .into_iter()
+        let mut names: Vec<String> = keys.into_iter()
             .filter_map(|k| k.strip_prefix(prefix).map(|x| x.to_string()))
             .collect();
         names.sort();
         Ok(names)
     };
 
-    let encpub   = match pull("age:key:*",      "age:key:")      { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
-    let encpriv  = match pull("age:privkey:*",  "age:privkey:")  { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
-    let signpub  = match pull("age:signpub:*",  "age:signpub:")  { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
-    let signpriv = match pull("age:signpriv:*", "age:signpriv:") { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
+    let encpub  = match pull("age:key:*",      "age:key:")      { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
+    let encpriv = match pull("age:privkey:*",  "age:privkey:")  { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
+    let signpub = match pull("age:signpub:*",  "age:signpub:")  { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
+    let signpriv= match pull("age:signpriv:*", "age:signpriv:") { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
 
-    let mut set: HashSet<String> = HashSet::new();
-    for n in encpub.into_iter().chain(encpriv).chain(signpub).chain(signpriv) {
-        set.insert(n);
-    }
+    let to_arr = |label: &str, v: Vec<String>| {
+        let mut out = vec![Protocol::BulkString(label.to_string())];
+        out.push(Protocol::Array(v.into_iter().map(Protocol::BulkString).collect()));
+        Protocol::Array(out)
+    };
 
-    let mut names: Vec<String> = set.into_iter().collect();
-    names.sort();
-
-    Protocol::Array(names.into_iter().map(Protocol::BulkString).collect())
+    Protocol::Array(vec![
+        to_arr("encpub", encpub),
+        to_arr("encpriv", encpriv),
+        to_arr("signpub", signpub),
+        to_arr("signpriv", signpriv),
+    ])
 }

src/crypto.rs

@@ -1,8 +1,8 @@
 use chacha20poly1305::{
-    aead::{Aead, KeyInit},
+    aead::{Aead, KeyInit, OsRng},
     XChaCha20Poly1305, XNonce,
 };
-use rand::{rngs::OsRng, RngCore};
+use rand::RngCore;
 use sha2::{Digest, Sha256};
 
 const VERSION: u8 = 1;
@@ -31,7 +31,7 @@ pub struct CryptoFactory {
 impl CryptoFactory {
     /// Accepts any secret bytes; turns them into a 32-byte key (SHA-256).
     pub fn new<S: AsRef<[u8]>>(secret: S) -> Self {
-        let mut h = Sha256::default();
+        let mut h = Sha256::new();
         h.update(b"xchacha20poly1305-factory:v1"); // domain separation
         h.update(secret.as_ref());
         let digest = h.finalize(); // 32 bytes

src/embedding.rs

@@ -1,405 +0,0 @@
-// Embedding abstraction and minimal providers.
-
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use serde::{Deserialize, Serialize};
-
-use crate::error::DBError;
-
-// Networking for OpenAI/Azure
-use std::time::Duration;
-use ureq::{Agent, AgentBuilder};
-use serde_json::json;
-
-/// Provider identifiers. Extend as needed to mirror LanceDB-supported providers.
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum EmbeddingProvider {
-    // Deterministic, local-only embedder for CI and offline development (text).
-    TestHash,
-    // Deterministic, local-only embedder for CI and offline development (image).
-    ImageTestHash,
-    // Placeholders for LanceDB-supported providers; implementers can add concrete backends later.
-    LanceFastEmbed,
-    LanceOpenAI,
-    LanceOther(String),
-}
-
-/// Serializable embedding configuration.
-/// params: arbitrary key-value map for provider-specific knobs (e.g., "dim", "api_key_env", etc.)
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct EmbeddingConfig {
-    pub provider: EmbeddingProvider,
-    pub model: String,
-    #[serde(default)]
-    pub params: HashMap<String, String>,
-}
-
-impl EmbeddingConfig {
-    pub fn get_param_usize(&self, key: &str) -> Option<usize> {
-        self.params.get(key).and_then(|v| v.parse::<usize>().ok())
-    }
-    pub fn get_param_string(&self, key: &str) -> Option<String> {
-        self.params.get(key).cloned()
-    }
-}
-
-/// A provider-agnostic text embedding interface.
-pub trait Embedder: Send + Sync {
-    /// Human-readable provider/model name
-    fn name(&self) -> String;
-    /// Embedding dimension
-    fn dim(&self) -> usize;
-    /// Embed a single text string into a fixed-length vector
-    fn embed(&self, text: &str) -> Result<Vec<f32>, DBError>;
-    /// Embed many texts; default maps embed() over inputs
-    fn embed_many(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, DBError> {
-        texts.iter().map(|t| self.embed(t)).collect()
-    }
-}
-
-//// ----------------------------- TEXT: deterministic test embedder -----------------------------
-
-/// Deterministic, no-deps, no-network embedder for CI and offline dev.
-/// Algorithm:
-/// - Fold bytes of UTF-8 into 'dim' buckets with a simple rolling hash
-/// - Apply tanh-like scaling and L2-normalize to unit length
-pub struct TestHashEmbedder {
-    dim: usize,
-    model_name: String,
-}
-
-impl TestHashEmbedder {
-    pub fn new(dim: usize, model_name: impl Into<String>) -> Self {
-        Self { dim, model_name: model_name.into() }
-    }
-
-    fn l2_normalize(mut v: Vec<f32>) -> Vec<f32> {
-        let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
-        if norm > 0.0 {
-            for x in &mut v {
-                *x /= norm;
-            }
-        }
-        v
-    }
-}
-
-impl Embedder for TestHashEmbedder {
-    fn name(&self) -> String {
-        format!("test-hash:{}", self.model_name)
-    }
-
-    fn dim(&self) -> usize {
-        self.dim
-    }
-
-    fn embed(&self, text: &str) -> Result<Vec<f32>, DBError> {
-        let mut acc = vec![0f32; self.dim];
-        // A simple, deterministic folding hash over bytes
-        let mut h1: u32 = 2166136261u32; // FNV-like seed
-        let mut h2: u32 = 0x9e3779b9u32; // golden ratio
-        for (i, b) in text.as_bytes().iter().enumerate() {
-            h1 ^= *b as u32;
-            h1 = h1.wrapping_mul(16777619u32);
-            h2 = h2.wrapping_add(((*b as u32) << (i % 13)) ^ (h1.rotate_left((i % 7) as u32)));
-            let idx = (h1 ^ h2) as usize % self.dim;
-            // Map byte to [-1, 1] and accumulate with mild decay by position
-            let val = ((*b as f32) / 127.5 - 1.0) * (1.0 / (1.0 + (i as f32 / 32.0)));
-            acc[idx] += val;
-        }
-        // Non-linear squashing to stabilize + normalize
-        for x in &mut acc {
-            *x = x.tanh();
-        }
-        Ok(Self::l2_normalize(acc))
-    }
-}
-
-//// ----------------------------- IMAGE: trait + deterministic test embedder -----------------------------
-
-/// Image embedding interface (separate from text to keep modality-specific inputs).
-pub trait ImageEmbedder: Send + Sync {
-    /// Human-readable provider/model name
-    fn name(&self) -> String;
-    /// Embedding dimension
-    fn dim(&self) -> usize;
-    /// Embed a single image (raw bytes)
-    fn embed_image(&self, bytes: &[u8]) -> Result<Vec<f32>, DBError>;
-    /// Embed many images; default maps embed_image() over inputs
-    fn embed_many_images(&self, images: &[Vec<u8>]) -> Result<Vec<Vec<f32>>, DBError> {
-        images.iter().map(|b| self.embed_image(b)).collect()
-    }
-}
-
-/// Deterministic image embedder that folds bytes into buckets, applies tanh-like nonlinearity,
-/// and L2-normalizes. Suitable for CI and offline development.
-/// NOTE: This is NOT semantic; it is a stable hash-like representation.
-pub struct TestImageHashEmbedder {
-    dim: usize,
-    model_name: String,
-}
-
-impl TestImageHashEmbedder {
-    pub fn new(dim: usize, model_name: impl Into<String>) -> Self {
-        Self { dim, model_name: model_name.into() }
-    }
-
-    fn l2_normalize(mut v: Vec<f32>) -> Vec<f32> {
-        let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
-        if norm > 0.0 {
-            for x in &mut v {
-                *x /= norm;
-            }
-        }
-        v
-    }
-}
-
-impl ImageEmbedder for TestImageHashEmbedder {
-    fn name(&self) -> String {
-        format!("test-image-hash:{}", self.model_name)
-    }
-
-    fn dim(&self) -> usize {
-        self.dim
-    }
-
-    fn embed_image(&self, bytes: &[u8]) -> Result<Vec<f32>, DBError> {
-        // Deterministic fold across bytes with two rolling accumulators.
-        let mut acc = vec![0f32; self.dim];
-        let mut h1: u32 = 0x811C9DC5; // FNV-like
-        let mut h2: u32 = 0x9E3779B9; // golden ratio
-        for (i, b) in bytes.iter().enumerate() {
-            h1 ^= *b as u32;
-            h1 = h1.wrapping_mul(16777619u32);
-            // combine with position and h2
-            h2 = h2.wrapping_add(((i as u32).rotate_left((i % 13) as u32)) ^ h1.rotate_left((i % 7) as u32));
-            let idx = (h1 ^ h2) as usize % self.dim;
-            // Map to [-1,1] and decay with position
-            let val = ((*b as f32) / 127.5 - 1.0) * (1.0 / (1.0 + (i as f32 / 128.0)));
-            acc[idx] += val;
-        }
-        for x in &mut acc {
-            *x = x.tanh();
-        }
-        Ok(Self::l2_normalize(acc))
-    }
-}
-
-//// OpenAI embedder (supports OpenAI and Azure OpenAI via REST)
-struct OpenAIEmbedder {
-    model: String,
-    dim: usize,
-    agent: Agent,
-    endpoint: String,
-    headers: Vec<(String, String)>,
-    use_azure: bool,
-}
-
-impl OpenAIEmbedder {
-    fn new_from_config(cfg: &EmbeddingConfig) -> Result<Self, DBError> {
-        // Whether to use Azure OpenAI
-        let use_azure = cfg
-            .get_param_string("use_azure")
-            .map(|s| s.eq_ignore_ascii_case("true"))
-            .unwrap_or(false);
-
-        // Resolve API key (OPENAI_API_KEY or AZURE_OPENAI_API_KEY by default)
-        let api_key_env = cfg
-            .get_param_string("api_key_env")
-            .unwrap_or_else(|| {
-                if use_azure {
-                    "AZURE_OPENAI_API_KEY".to_string()
-                } else {
-                    "OPENAI_API_KEY".to_string()
-                }
-            });
-        let api_key = std::env::var(&api_key_env)
-            .map_err(|_| DBError(format!("Missing API key in env '{}'", api_key_env)))?;
-
-        // Resolve endpoint
-        // - Standard OpenAI: https://api.openai.com/v1/embeddings (default) or params["base_url"]
-        // - Azure OpenAI: {azure_endpoint}/openai/deployments/{deployment}/embeddings?api-version=...
-        let endpoint = if use_azure {
-            let base = cfg
-                .get_param_string("azure_endpoint")
-                .ok_or_else(|| DBError("Missing 'azure_endpoint' for Azure OpenAI".into()))?;
-            let deployment = cfg
-                .get_param_string("azure_deployment")
-                .unwrap_or_else(|| cfg.model.clone());
-            let api_version = cfg
-                .get_param_string("azure_api_version")
-                .unwrap_or_else(|| "2023-05-15".to_string());
-            format!(
-                "{}/openai/deployments/{}/embeddings?api-version={}",
-                base.trim_end_matches('/'),
-                deployment,
-                api_version
-            )
-        } else {
-            cfg.get_param_string("base_url")
-                .unwrap_or_else(|| "https://api.openai.com/v1/embeddings".to_string())
-        };
-
-        // Determine expected dimension (default 1536 for text-embedding-3-small; callers should override if needed)
-        let dim = cfg
-            .get_param_usize("dim")
-            .or_else(|| cfg.get_param_usize("dimensions"))
-            .unwrap_or(1536);
-
-        // Build an HTTP agent with timeouts (blocking; no tokio runtime involved)
-        let agent = AgentBuilder::new()
-            .timeout_read(Duration::from_secs(30))
-            .timeout_write(Duration::from_secs(30))
-            .build();
-
-        // Headers
-        let mut headers: Vec<(String, String)> = Vec::new();
-        headers.push(("Content-Type".to_string(), "application/json".to_string()));
-        if use_azure {
-            headers.push(("api-key".to_string(), api_key));
-        } else {
-            headers.push(("Authorization".to_string(), format!("Bearer {}", api_key)));
-        }
-
-        Ok(Self {
-            model: cfg.model.clone(),
-            dim,
-            agent,
-            endpoint,
-            headers,
-            use_azure,
-        })
-    }
-
-    fn request_many(&self, inputs: &[String]) -> Result<Vec<Vec<f32>>, DBError> {
-        // Compose request body:
-        // - Standard OpenAI: { "model": ..., "input": [...], "dimensions": dim? }
-        // - Azure:           { "input": [...], "dimensions": dim? } (model from deployment)
-        let mut body = if self.use_azure {
-            json!({ "input": inputs })
-        } else {
-            json!({ "model": self.model, "input": inputs })
-        };
-        if self.dim > 0 {
-            body.as_object_mut()
-                .unwrap()
-                .insert("dimensions".to_string(), json!(self.dim));
-        }
-
-        // Build request
-        let mut req = self.agent.post(&self.endpoint);
-        for (k, v) in &self.headers {
-            req = req.set(k, v);
-        }
-
-        // Send and handle errors
-        let resp = req.send_json(body);
-        let text = match resp {
-            Ok(r) => r
-                .into_string()
-                .map_err(|e| DBError(format!("Failed to read embeddings response: {}", e)))?,
-            Err(ureq::Error::Status(code, r)) => {
-                let body = r.into_string().unwrap_or_default();
-                return Err(DBError(format!("Embeddings API error {}: {}", code, body)));
-            }
-            Err(e) => return Err(DBError(format!("HTTP request failed: {}", e))),
-        };
-
-        let val: serde_json::Value = serde_json::from_str(&text)
-            .map_err(|e| DBError(format!("Invalid JSON from embeddings API: {}", e)))?;
-
-        let data = val
-            .get("data")
-            .and_then(|d| d.as_array())
-            .ok_or_else(|| DBError("Embeddings API response missing 'data' array".into()))?;
-
-        let mut out: Vec<Vec<f32>> = Vec::with_capacity(data.len());
-        for item in data {
-            let emb = item
-                .get("embedding")
-                .and_then(|e| e.as_array())
-                .ok_or_else(|| DBError("Embeddings API item missing 'embedding'".into()))?;
-            let mut v: Vec<f32> = Vec::with_capacity(emb.len());
-            for n in emb {
-                let f = n
-                    .as_f64()
-                    .ok_or_else(|| DBError("Embedding element is not a number".into()))?;
-                v.push(f as f32);
-            }
-            if self.dim > 0 && v.len() != self.dim {
-                return Err(DBError(format!(
-                    "Embedding dimension mismatch: expected {}, got {}. Configure 'dim' or 'dimensions' to match output.",
-                    self.dim, v.len()
-                )));
-            }
-            out.push(v);
-        }
-        Ok(out)
-    }
-}
-
-impl Embedder for OpenAIEmbedder {
-    fn name(&self) -> String {
-        if self.use_azure {
-            format!("azure-openai:{}", self.model)
-        } else {
-            format!("openai:{}", self.model)
-        }
-    }
-
-    fn dim(&self) -> usize {
-        self.dim
-    }
-
-    fn embed(&self, text: &str) -> Result<Vec<f32>, DBError> {
-        let v = self.request_many(&[text.to_string()])?;
-        Ok(v.into_iter().next().unwrap_or_else(|| vec![0.0; self.dim]))
-    }
-
-    fn embed_many(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, DBError> {
-        if texts.is_empty() {
-            return Ok(vec![]);
-        }
-        self.request_many(texts)
-    }
-}
-
-/// Create an embedder instance from a config.
-/// - TestHash: uses params["dim"] or defaults to 64
-/// - LanceOpenAI: uses OpenAI (or Azure OpenAI) embeddings REST API
-/// - Other Lance providers can be added similarly
-pub fn create_embedder(config: &EmbeddingConfig) -> Result<Arc<dyn Embedder>, DBError> {
-    match &config.provider {
-        EmbeddingProvider::TestHash => {
-            let dim = config.get_param_usize("dim").unwrap_or(64);
-            Ok(Arc::new(TestHashEmbedder::new(dim, config.model.clone())))
-        }
-        EmbeddingProvider::LanceOpenAI => {
-            let inner = OpenAIEmbedder::new_from_config(config)?;
-            Ok(Arc::new(inner))
-        }
-        EmbeddingProvider::ImageTestHash => {
-            Err(DBError("Use create_image_embedder() for image providers".into()))
-        }
-        EmbeddingProvider::LanceFastEmbed => Err(DBError("LanceFastEmbed provider not yet implemented in Rust embedding layer; configure 'test-hash' or use 'openai'".into())),
-        EmbeddingProvider::LanceOther(p) => Err(DBError(format!("Lance provider '{}' not implemented; configure 'openai' or 'test-hash'", p))),
-    }
-}
-
-/// Create an image embedder instance from a config.
-pub fn create_image_embedder(config: &EmbeddingConfig) -> Result<Arc<dyn ImageEmbedder>, DBError> {
-    match &config.provider {
-        EmbeddingProvider::ImageTestHash => {
-            let dim = config.get_param_usize("dim").unwrap_or(512);
-            Ok(Arc::new(TestImageHashEmbedder::new(dim, config.model.clone())))
-        }
-        EmbeddingProvider::TestHash | EmbeddingProvider::LanceOpenAI => {
-            Err(DBError("Configured text provider; dataset expects image provider (e.g., 'testimagehash')".into()))
-        }
-        EmbeddingProvider::LanceFastEmbed => Err(DBError("Image provider 'lancefastembed' not yet implemented".into())),
-        EmbeddingProvider::LanceOther(p) => Err(DBError(format!("Image provider '{}' not implemented; use 'testimagehash' for now", p))),
-    }
-}

src/error.rs

@@ -9,6 +9,12 @@ use bincode;
 #[derive(Debug)]
 pub struct DBError(pub String);
 
+impl std::fmt::Display for DBError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
 impl From<std::io::Error> for DBError {
     fn from(item: std::io::Error) -> Self {
         DBError(item.to_string().clone())
@@ -92,3 +98,40 @@ impl From<chacha20poly1305::Error> for DBError {
         DBError(item.to_string())
     }
 }
+
+// Lance and related dependencies error handling
+impl From<lance::Error> for DBError {
+    fn from(item: lance::Error) -> Self {
+        DBError(item.to_string())
+    }
+}
+
+impl From<arrow::error::ArrowError> for DBError {
+    fn from(item: arrow::error::ArrowError) -> Self {
+        DBError(item.to_string())
+    }
+}
+
+impl From<reqwest::Error> for DBError {
+    fn from(item: reqwest::Error) -> Self {
+        DBError(item.to_string())
+    }
+}
+
+impl From<image::ImageError> for DBError {
+    fn from(item: image::ImageError) -> Self {
+        DBError(item.to_string())
+    }
+}
+
+impl From<uuid::Error> for DBError {
+    fn from(item: uuid::Error) -> Self {
+        DBError(item.to_string())
+    }
+}
+
+impl From<base64::DecodeError> for DBError {
+    fn from(item: base64::DecodeError) -> Self {
+        DBError(item.to_string())
+    }
+}

src/lib.rs

@@ -1,18 +1,11 @@
-pub mod age;
-pub mod sym;
+pub mod age;   // NEW
 pub mod cmd;
 pub mod crypto;
 pub mod error;
+pub mod lance_store;    // Add Lance store module
 pub mod options;
 pub mod protocol;
-pub mod rpc;
-pub mod rpc_server;
 pub mod server;
 pub mod storage;
-pub mod storage_trait;
-pub mod storage_sled;
-pub mod admin_meta;
-pub mod tantivy_search;
-pub mod search_cmd;
-pub mod lance_store;
-pub mod embedding;
+pub mod storage_trait;  // Add this
+pub mod storage_sled;   // Add this

src/main.rs

@@ -1,10 +1,8 @@
 // #![allow(unused_imports)]
 
-use std::path::PathBuf;
 use tokio::net::TcpListener;
 
 use herodb::server;
-use herodb::rpc_server;
 
 use clap::Parser;
 
@@ -14,7 +12,7 @@ use clap::Parser;
 struct Args {
     /// The directory of Redis DB file
     #[arg(long)]
-    dir: PathBuf,
+    dir: String,
 
     /// The port of the Redis server, default is 6379 if not specified
     #[arg(long)]
@@ -24,29 +22,18 @@ struct Args {
     #[arg(long)]
     debug: bool,
 
-    /// Master encryption key for encrypted databases (deprecated; ignored for data DBs)
+
+    /// Master encryption key for encrypted databases
     #[arg(long)]
     encryption_key: Option<String>,
 
-    /// Encrypt the database (deprecated; ignored for data DBs)
+    /// Encrypt the database
     #[arg(long)]
     encrypt: bool,
 
-    /// Enable RPC management server
-    #[arg(long)]
-    enable_rpc: bool,
-
-    /// RPC server port (default: 8080)
-    #[arg(long, default_value = "8080")]
-    rpc_port: u16,
-
     /// Use the sled backend
     #[arg(long)]
     sled: bool,
-
-    /// Admin secret used to encrypt DB 0 and authorize admin access (required)
-    #[arg(long)]
-    admin_secret: String,
 }
 
 #[tokio::main]
@@ -61,19 +48,9 @@ async fn main() {
         .await
         .unwrap();
 
-    // deprecation warnings for legacy flags
-    if args.encrypt || args.encryption_key.is_some() {
-        eprintln!("warning: --encrypt and --encryption-key are deprecated and ignored for data DBs. Admin DB 0 is always encrypted with --admin-secret.");
-    }
-    // basic validation for admin secret
-    if args.admin_secret.trim().is_empty() {
-        eprintln!("error: --admin-secret must not be empty");
-        std::process::exit(2);
-    }
-
     // new DB option
     let option = herodb::options::DBOption {
-        dir: args.dir.clone(),
+        dir: args.dir,
         port,
         debug: args.debug,
         encryption_key: args.encryption_key,
@@ -83,42 +60,14 @@ async fn main() {
         } else {
             herodb::options::BackendType::Redb
         },
-        admin_secret: args.admin_secret.clone(),
     };
 
-    let backend = option.backend.clone();
-
-    // Bootstrap admin DB 0 before opening any server storage
-    if let Err(e) = herodb::admin_meta::ensure_bootstrap(&args.dir, backend.clone(), &args.admin_secret) {
-        eprintln!("Failed to bootstrap admin DB 0: {}", e.0);
-        std::process::exit(2);
-    }
-
     // new server
     let server = server::Server::new(option).await;
 
     // Add a small delay to ensure the port is ready
     tokio::time::sleep(std::time::Duration::from_millis(100)).await;
 
-    // Start RPC server if enabled
-    let _rpc_handle = if args.enable_rpc {
-        let rpc_addr = format!("127.0.0.1:{}", args.rpc_port).parse().unwrap();
-        let base_dir = args.dir.clone();
-
-        match rpc_server::start_rpc_server(rpc_addr, base_dir, backend, args.admin_secret.clone()).await {
-            Ok(handle) => {
-                println!("RPC management server started on port {}", args.rpc_port);
-                Some(handle)
-            }
-            Err(e) => {
-                eprintln!("Failed to start RPC server: {}", e);
-                None
-            }
-        }
-    } else {
-        None
-    };
-
     // accept new connections
     loop {
         let stream = listener.accept().await;

src/options.rs

@@ -1,23 +1,15 @@
-use std::path::PathBuf;
-
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone)]
 pub enum BackendType {
     Redb,
     Sled,
-    Tantivy, // Full-text search backend (no KV storage)
-    Lance,   // Vector database backend (no KV storage)
 }
 
 #[derive(Debug, Clone)]
 pub struct DBOption {
-    pub dir: PathBuf,
+    pub dir: String,
     pub port: u16,
     pub debug: bool,
-    // Deprecated for data DBs; retained for backward-compat on CLI parsing
     pub encrypt: bool,
-    // Deprecated for data DBs; retained for backward-compat on CLI parsing
     pub encryption_key: Option<String>,
     pub backend: BackendType,
-    // New: required admin secret, used to encrypt DB 0 and authorize admin operations
-    pub admin_secret: String,
 }

src/rpc_server.rs

@@ -1,50 +0,0 @@
-use std::net::SocketAddr;
-use std::path::PathBuf;
-use jsonrpsee::server::{ServerBuilder, ServerHandle};
-use jsonrpsee::RpcModule;
-
-use crate::rpc::{RpcServer, RpcServerImpl};
-
-/// Start the RPC server on the specified address
-pub async fn start_rpc_server(addr: SocketAddr, base_dir: PathBuf, backend: crate::options::BackendType, admin_secret: String) -> Result<ServerHandle, Box<dyn std::error::Error + Send + Sync>> {
-    // Create the RPC server implementation
-    let rpc_impl = RpcServerImpl::new(base_dir, backend, admin_secret);
-
-    // Create the RPC module
-    let mut module = RpcModule::new(());
-    module.merge(RpcServer::into_rpc(rpc_impl))?;
-
-    // Build the server with both HTTP and WebSocket support
-    let server = ServerBuilder::default()
-        .build(addr)
-        .await?;
-
-    // Start the server
-    let handle = server.start(module);
-
-    println!("RPC server started on {}", addr);
-
-    Ok(handle)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::time::Duration;
-
-    #[tokio::test]
-    async fn test_rpc_server_startup() {
-        let addr = "127.0.0.1:0".parse().unwrap(); // Use port 0 for auto-assignment
-        let base_dir = PathBuf::from("/tmp/test_rpc");
-        let backend = crate::options::BackendType::Redb; // Default for test
-
-        let handle = start_rpc_server(addr, base_dir, backend, "test-admin".to_string()).await.unwrap();
-
-        // Give the server a moment to start
-        tokio::time::sleep(Duration::from_millis(100)).await;
-
-        // Stop the server
-        handle.stop().unwrap();
-        handle.stopped().await;
-    }
-}

src/search_cmd.rs

@@ -1,378 +0,0 @@
-use crate::{
-    error::DBError,
-    protocol::Protocol,
-    server::Server,
-    tantivy_search::{
-        FieldDef, Filter, FilterType, IndexConfig, NumericType, SearchOptions, TantivySearch,
-    },
-};
-use std::collections::HashMap;
-use std::sync::Arc;
-
-pub async fn ft_create_cmd(
-    server: &Server,
-    index_name: String,
-    schema: Vec<(String, String, Vec<String>)>,
-) -> Result<Protocol, DBError> {
-    if server.selected_db == 0 {
-        return Ok(Protocol::err("FT commands are not allowed on DB 0"));
-    }
-    // Enforce Tantivy backend for selected DB
-    let is_tantivy = crate::admin_meta::get_database_backend(
-        &server.option.dir,
-        server.option.backend.clone(),
-        &server.option.admin_secret,
-        server.selected_db,
-    )
-    .ok()
-    .flatten()
-    .map(|b| matches!(b, crate::options::BackendType::Tantivy))
-    .unwrap_or(false);
-    if !is_tantivy {
-        return Ok(Protocol::err("ERR DB backend is not Tantivy; FT.* commands are not allowed"));
-    }
-
-    if !server.has_write_permission() {
-        return Ok(Protocol::err("ERR write permission denied"));
-    }
-
-    // Parse schema into field definitions
-    let mut field_definitions = Vec::new();
-    for (field_name, field_type, options) in schema {
-        let field_def = match field_type.to_uppercase().as_str() {
-            "TEXT" => {
-                let mut sortable = false;
-                let mut no_index = false;
-                // Weight is not used in current implementation
-                let mut _weight = 1.0f32;
-                let mut i = 0;
-                while i < options.len() {
-                    match options[i].to_uppercase().as_str() {
-                        "WEIGHT" => {
-                            if i + 1 < options.len() {
-                                _weight = options[i + 1].parse::<f32>().unwrap_or(1.0);
-                                i += 2;
-                                continue;
-                            }
-                        }
-                        "SORTABLE" => {
-                            sortable = true;
-                        }
-                        "NOINDEX" => {
-                            no_index = true;
-                        }
-                        _ => {}
-                    }
-                    i += 1;
-                }
-                FieldDef::Text {
-                    stored: true,
-                    indexed: !no_index,
-                    tokenized: true,
-                    fast: sortable,
-                }
-            }
-            "NUMERIC" => {
-                // default to F64
-                let mut sortable = false;
-                for opt in &options {
-                    if opt.to_uppercase() == "SORTABLE" {
-                        sortable = true;
-                    }
-                }
-                FieldDef::Numeric {
-                    stored: true,
-                    indexed: true,
-                    fast: sortable,
-                    precision: NumericType::F64,
-                }
-            }
-            "TAG" => {
-                let mut separator = ",".to_string();
-                let mut case_sensitive = false;
-                let mut i = 0;
-                while i < options.len() {
-                    match options[i].to_uppercase().as_str() {
-                        "SEPARATOR" => {
-                            if i + 1 < options.len() {
-                                separator = options[i + 1].clone();
-                                i += 2;
-                                continue;
-                            }
-                        }
-                        "CASESENSITIVE" => {
-                            case_sensitive = true;
-                        }
-                        _ => {}
-                    }
-                    i += 1;
-                }
-                FieldDef::Tag {
-                    stored: true,
-                    separator,
-                    case_sensitive,
-                }
-            }
-            "GEO" => FieldDef::Geo { stored: true },
-            _ => {
-                return Err(DBError(format!("Unknown field type: {}", field_type)));
-            }
-        };
-        field_definitions.push((field_name, field_def));
-    }
-
-    // Create the search index
-    let search_path = server.search_index_path();
-    let config = IndexConfig::default();
-    let search_index = TantivySearch::new_with_schema(
-        search_path,
-        index_name.clone(),
-        field_definitions,
-        Some(config),
-    )?;
-
-    // Store in registry
-    let mut indexes = server.search_indexes.write().unwrap();
-    indexes.insert(index_name, Arc::new(search_index));
-
-    Ok(Protocol::SimpleString("OK".to_string()))
-}
-
-pub async fn ft_add_cmd(
-    server: &Server,
-    index_name: String,
-    doc_id: String,
-    _score: f64,
-    fields: HashMap<String, String>,
-) -> Result<Protocol, DBError> {
-    if server.selected_db == 0 {
-        return Ok(Protocol::err("FT commands are not allowed on DB 0"));
-    }
-    // Enforce Tantivy backend for selected DB
-    let is_tantivy = crate::admin_meta::get_database_backend(
-        &server.option.dir,
-        server.option.backend.clone(),
-        &server.option.admin_secret,
-        server.selected_db,
-    )
-    .ok()
-    .flatten()
-    .map(|b| matches!(b, crate::options::BackendType::Tantivy))
-    .unwrap_or(false);
-    if !is_tantivy {
-        return Ok(Protocol::err("ERR DB backend is not Tantivy; FT.* commands are not allowed"));
-    }
-    if !server.has_write_permission() {
-        return Ok(Protocol::err("ERR write permission denied"));
-    }
-    let indexes = server.search_indexes.read().unwrap();
-    let search_index = indexes
-        .get(&index_name)
-        .ok_or_else(|| DBError(format!("Index '{}' not found", index_name)))?;
-    search_index.add_document_with_fields(&doc_id, fields)?;
-    Ok(Protocol::SimpleString("OK".to_string()))
-}
-
-pub async fn ft_search_cmd(
-    server: &Server,
-    index_name: String,
-    query: String,
-    filters: Vec<(String, String)>,
-    limit: Option<usize>,
-    offset: Option<usize>,
-    return_fields: Option<Vec<String>>,
-) -> Result<Protocol, DBError> {
-    if server.selected_db == 0 {
-        return Ok(Protocol::err("FT commands are not allowed on DB 0"));
-    }
-    // Enforce Tantivy backend for selected DB
-    let is_tantivy = crate::admin_meta::get_database_backend(
-        &server.option.dir,
-        server.option.backend.clone(),
-        &server.option.admin_secret,
-        server.selected_db,
-    )
-    .ok()
-    .flatten()
-    .map(|b| matches!(b, crate::options::BackendType::Tantivy))
-    .unwrap_or(false);
-    if !is_tantivy {
-        return Ok(Protocol::err("ERR DB backend is not Tantivy; FT.* commands are not allowed"));
-    }
-    if !server.has_read_permission() {
-        return Ok(Protocol::err("ERR read permission denied"));
-    }
-    let indexes = server.search_indexes.read().unwrap();
-    let search_index = indexes
-        .get(&index_name)
-        .ok_or_else(|| DBError(format!("Index '{}' not found", index_name)))?;
-
-    let search_filters = filters
-        .into_iter()
-        .map(|(field, value)| Filter {
-            field,
-            filter_type: FilterType::Equals(value),
-        })
-        .collect();
-
-    let options = SearchOptions {
-        limit: limit.unwrap_or(10),
-        offset: offset.unwrap_or(0),
-        filters: search_filters,
-        sort_by: None,
-        return_fields,
-        highlight: false,
-    };
-
-    let results = search_index.search_with_options(&query, options)?;
- 
-    // Format results as a flattened Redis protocol array to match client expectations:
-    // [ total, doc_id, score, field, value, field, value, ... , doc_id, score, ... ]
-    let mut response = Vec::new();
-    // First element is the total count
-    response.push(Protocol::BulkString(results.total.to_string()));
-    // Then each document flattened
-    for mut doc in results.documents {
-        // Add document ID if it exists
-        if let Some(id) = doc.fields.get("_id") {
-            response.push(Protocol::BulkString(id.clone()));
-        }
-        // Add score
-        response.push(Protocol::BulkString(doc.score.to_string()));
-        // Add fields as key-value pairs
-        for (field_name, field_value) in std::mem::take(&mut doc.fields) {
-            if field_name != "_id" {
-                response.push(Protocol::BulkString(field_name));
-                response.push(Protocol::BulkString(field_value));
-            }
-        }
-    }
- 
-    Ok(Protocol::Array(response))
-}
-
-pub async fn ft_del_cmd(
-    server: &Server,
-    index_name: String,
-    doc_id: String,
-) -> Result<Protocol, DBError> {
-    if server.selected_db == 0 {
-        return Ok(Protocol::err("FT commands are not allowed on DB 0"));
-    }
-    // Enforce Tantivy backend for selected DB
-    let is_tantivy = crate::admin_meta::get_database_backend(
-        &server.option.dir,
-        server.option.backend.clone(),
-        &server.option.admin_secret,
-        server.selected_db,
-    )
-    .ok()
-    .flatten()
-    .map(|b| matches!(b, crate::options::BackendType::Tantivy))
-    .unwrap_or(false);
-    if !is_tantivy {
-        return Ok(Protocol::err("ERR DB backend is not Tantivy; FT.* commands are not allowed"));
-    }
-    if !server.has_write_permission() {
-        return Ok(Protocol::err("ERR write permission denied"));
-    }
-    let indexes = server.search_indexes.read().unwrap();
-    let search_index = indexes
-        .get(&index_name)
-        .ok_or_else(|| DBError(format!("Index '{}' not found", index_name)))?;
-    let existed = search_index.delete_document_by_id(&doc_id)?;
-    Ok(Protocol::SimpleString(if existed { "1".to_string() } else { "0".to_string() }))
-}
-
-pub async fn ft_info_cmd(server: &Server, index_name: String) -> Result<Protocol, DBError> {
-    if server.selected_db == 0 {
-        return Ok(Protocol::err("FT commands are not allowed on DB 0"));
-    }
-    // Enforce Tantivy backend for selected DB
-    let is_tantivy = crate::admin_meta::get_database_backend(
-        &server.option.dir,
-        server.option.backend.clone(),
-        &server.option.admin_secret,
-        server.selected_db,
-    )
-    .ok()
-    .flatten()
-    .map(|b| matches!(b, crate::options::BackendType::Tantivy))
-    .unwrap_or(false);
-    if !is_tantivy {
-        return Ok(Protocol::err("ERR DB backend is not Tantivy; FT.* commands are not allowed"));
-    }
-    if !server.has_read_permission() {
-        return Ok(Protocol::err("ERR read permission denied"));
-    }
-    let indexes = server.search_indexes.read().unwrap();
-    let search_index = indexes
-        .get(&index_name)
-        .ok_or_else(|| DBError(format!("Index '{}' not found", index_name)))?;
-    let info = search_index.get_info()?;
-
-    // Format info as Redis protocol
-    let mut response = Vec::new();
-    response.push(Protocol::BulkString("index_name".to_string()));
-    response.push(Protocol::BulkString(info.name));
-    response.push(Protocol::BulkString("num_docs".to_string()));
-    response.push(Protocol::BulkString(info.num_docs.to_string()));
-    response.push(Protocol::BulkString("num_fields".to_string()));
-    response.push(Protocol::BulkString(info.fields.len().to_string()));
-    response.push(Protocol::BulkString("fields".to_string()));
-    let fields_str = info
-        .fields
-        .iter()
-        .map(|f| format!("{}:{}", f.name, f.field_type))
-        .collect::<Vec<_>>()
-        .join(", ");
-    response.push(Protocol::BulkString(fields_str));
-    Ok(Protocol::Array(response))
-}
-
-pub async fn ft_drop_cmd(server: &Server, index_name: String) -> Result<Protocol, DBError> {
-    if server.selected_db == 0 {
-        return Ok(Protocol::err("FT commands are not allowed on DB 0"));
-    }
-    // Enforce Tantivy backend for selected DB
-    let is_tantivy = crate::admin_meta::get_database_backend(
-        &server.option.dir,
-        server.option.backend.clone(),
-        &server.option.admin_secret,
-        server.selected_db,
-    )
-    .ok()
-    .flatten()
-    .map(|b| matches!(b, crate::options::BackendType::Tantivy))
-    .unwrap_or(false);
-    if !is_tantivy {
-        return Ok(Protocol::err("ERR DB backend is not Tantivy; FT.* commands are not allowed"));
-    }
-
-    if !server.has_write_permission() {
-        return Ok(Protocol::err("ERR write permission denied"));
-    }
-
-    // Remove from registry and files; report error if nothing to drop
-    let mut existed = false;
-    {
-        let mut indexes = server.search_indexes.write().unwrap();
-        if indexes.remove(&index_name).is_some() {
-            existed = true;
-        }
-    }
-
-    // Remove the index files from disk
-    let index_path = server.search_index_path().join(&index_name);
-    if index_path.exists() {
-        std::fs::remove_dir_all(&index_path)
-            .map_err(|e| DBError(format!("Failed to remove index files: {}", e)))?;
-        existed = true;
-    }
-
-    if !existed {
-        return Ok(Protocol::err(&format!("ERR Index '{}' not found", index_name)));
-    }
-
-    Ok(Protocol::SimpleString("OK".to_string()))
-}

src/server.rs

@@ -9,19 +9,12 @@ use std::sync::atomic::{AtomicU64, Ordering};
 
 use crate::cmd::Cmd;
 use crate::error::DBError;
+use crate::lance_store::LanceStore;
 use crate::options;
 use crate::protocol::Protocol;
+use crate::storage::Storage;
+use crate::storage_sled::SledStorage;
 use crate::storage_trait::StorageBackend;
-use crate::admin_meta;
-
-// Embeddings: config and cache
-use crate::embedding::{EmbeddingConfig, create_embedder, Embedder, create_image_embedder, ImageEmbedder};
-use serde_json;
-use ureq::{Agent, AgentBuilder};
-use std::time::Duration;
-use std::io::Read;
-
-const NO_DB_SELECTED: u64 = u64::MAX;
 
 #[derive(Clone)]
 pub struct Server {
@@ -30,23 +23,13 @@ pub struct Server {
     pub client_name: Option<String>,
     pub selected_db: u64, // Changed from usize to u64
     pub queued_cmd: Option<Vec<(Cmd, Protocol)>>,
-    pub current_permissions: Option<crate::rpc::Permissions>,
 
-    // In-memory registry of Tantivy search indexes for this server
-    pub search_indexes: Arc<std::sync::RwLock<HashMap<String, Arc<crate::tantivy_search::TantivySearch>>>>,
-
-    // Per-DB Lance stores (vector DB), keyed by db_id
-    pub lance_stores: Arc<std::sync::RwLock<HashMap<u64, Arc<crate::lance_store::LanceStore>>>>,
-
-    // Per-(db_id, dataset) embedder cache (text)
-    pub embedders: Arc<std::sync::RwLock<HashMap<(u64, String), Arc<dyn Embedder>>>>,
-
-    // Per-(db_id, dataset) image embedder cache (image)
-    pub image_embedders: Arc<std::sync::RwLock<HashMap<(u64, String), Arc<dyn ImageEmbedder>>>>,
-    
     // BLPOP waiter registry: per (db_index, key) FIFO of waiters
     pub list_waiters: Arc<Mutex<HashMap<u64, HashMap<String, Vec<Waiter>>>>>,
     pub waiter_seq: Arc<AtomicU64>,
+    
+    // Lance vector store
+    pub lance_store: Option<Arc<LanceStore>>,
 }
 
 pub struct Waiter {
@@ -63,326 +46,81 @@ pub enum PopSide {
 
 impl Server {
     pub async fn new(option: options::DBOption) -> Self {
+        // Initialize Lance store
+        let lance_data_dir = std::path::PathBuf::from(&option.dir).join("lance");
+        let lance_store = match LanceStore::new(lance_data_dir).await {
+            Ok(store) => Some(Arc::new(store)),
+            Err(e) => {
+                eprintln!("Warning: Failed to initialize Lance store: {}", e.0);
+                None
+            }
+        };
+
         Server {
             db_cache: Arc::new(std::sync::RwLock::new(HashMap::new())),
             option,
             client_name: None,
-            selected_db: NO_DB_SELECTED,
+            selected_db: 0,
             queued_cmd: None,
-            current_permissions: None,
-    
-            search_indexes: Arc::new(std::sync::RwLock::new(HashMap::new())),
-            lance_stores: Arc::new(std::sync::RwLock::new(HashMap::new())),
-            embedders: Arc::new(std::sync::RwLock::new(HashMap::new())),
-            image_embedders: Arc::new(std::sync::RwLock::new(HashMap::new())),
+
             list_waiters: Arc::new(Mutex::new(HashMap::new())),
             waiter_seq: Arc::new(AtomicU64::new(1)),
+            lance_store,
         }
     }
 
-    // Path where search indexes are stored, namespaced per selected DB:
-    // <base_dir>/search_indexes/<db_id>
-    pub fn search_index_path(&self) -> std::path::PathBuf {
-        let base = std::path::PathBuf::from(&self.option.dir)
-            .join("search_indexes")
-            .join(self.selected_db.to_string());
-        if !base.exists() {
-            let _ = std::fs::create_dir_all(&base);
-        }
-        base
-    }
-
-    // Path where Lance datasets are stored, namespaced per selected DB:
-    // <base_dir>/lance/<db_id>
-    pub fn lance_data_path(&self) -> std::path::PathBuf {
-        let base = std::path::PathBuf::from(&self.option.dir)
-            .join("lance")
-            .join(self.selected_db.to_string());
-        if !base.exists() {
-            let _ = std::fs::create_dir_all(&base);
-        }
-        base
+    pub fn lance_store(&self) -> Result<Arc<LanceStore>, DBError> {
+        self.lance_store
+            .as_ref()
+            .cloned()
+            .ok_or_else(|| DBError("Lance store not initialized".to_string()))
     }
 
     pub fn current_storage(&self) -> Result<Arc<dyn StorageBackend>, DBError> {
-        // Require explicit SELECT before any storage access
-        if self.selected_db == NO_DB_SELECTED {
-            return Err(DBError("No database selected. Use SELECT <id> [KEY <key>] first".to_string()));
-        }
-        // Admin DB 0 access must be authenticated with SELECT 0 KEY <admin_secret>
-        if self.selected_db == 0 {
-            if !matches!(self.current_permissions, Some(crate::rpc::Permissions::ReadWrite)) {
-                return Err(DBError("Admin DB 0 requires SELECT 0 KEY <admin_secret>".to_string()));
-            }
-        }
-
         let mut cache = self.db_cache.write().unwrap();
-
+        
         if let Some(storage) = cache.get(&self.selected_db) {
             return Ok(storage.clone());
         }
-
-        // Use process-wide shared handles to avoid sled/reDB double-open lock contention.
-        let storage = if self.selected_db == 0 {
-            // Admin DB 0: always via singleton
-            admin_meta::open_admin_storage(
-                &self.option.dir,
-                self.option.backend.clone(),
-                &self.option.admin_secret,
-            )?
-        } else {
-            // Data DBs: via global registry keyed by id
-            admin_meta::open_data_storage(
-                &self.option.dir,
-                self.option.backend.clone(),
-                &self.option.admin_secret,
-                self.selected_db,
-            )?
+        
+        
+        // Create new database file
+        let db_file_path = std::path::PathBuf::from(self.option.dir.clone())
+            .join(format!("{}.db", self.selected_db));
+        
+        // Ensure the directory exists before creating the database file
+        if let Some(parent_dir) = db_file_path.parent() {
+            std::fs::create_dir_all(parent_dir).map_err(|e| {
+                DBError(format!("Failed to create directory {}: {}", parent_dir.display(), e))
+            })?;
+        }
+        
+        println!("Creating new db file: {}", db_file_path.display());
+        
+        let storage: Arc<dyn StorageBackend> = match self.option.backend {
+            options::BackendType::Redb => {
+                Arc::new(Storage::new(
+                    db_file_path,
+                    self.should_encrypt_db(self.selected_db),
+                    self.option.encryption_key.as_deref()
+                )?)
+            }
+            options::BackendType::Sled => {
+                Arc::new(SledStorage::new(
+                    db_file_path,
+                    self.should_encrypt_db(self.selected_db),
+                    self.option.encryption_key.as_deref()
+                )?)
+            }
         };
-
+        
         cache.insert(self.selected_db, storage.clone());
         Ok(storage)
     }
-
-    /// Get or create the LanceStore for the currently selected DB.
-    /// Only valid for non-zero DBs and when the backend is Lance.
-    pub fn lance_store(&self) -> Result<Arc<crate::lance_store::LanceStore>, DBError> {
-        if self.selected_db == 0 {
-            return Err(DBError("Lance not available on admin DB 0".to_string()));
-        }
-        // Resolve backend for selected_db
-        let backend_opt = crate::admin_meta::get_database_backend(
-            &self.option.dir,
-            self.option.backend.clone(),
-            &self.option.admin_secret,
-            self.selected_db,
-        )
-        .ok()
-        .flatten();
-
-        if !matches!(backend_opt, Some(crate::options::BackendType::Lance)) {
-            return Err(DBError("ERR DB backend is not Lance; LANCE.* commands are not allowed".to_string()));
-        }
-
-        // Fast path: read lock
-        {
-            let map = self.lance_stores.read().unwrap();
-            if let Some(store) = map.get(&self.selected_db) {
-                return Ok(store.clone());
-            }
-        }
-
-        // Slow path: create and insert
-        let store = Arc::new(crate::lance_store::LanceStore::new(&self.option.dir, self.selected_db)?);
-        {
-            let mut map = self.lance_stores.write().unwrap();
-            map.insert(self.selected_db, store.clone());
-        }
-        Ok(store)
-    }
-
-    // ----- Embedding configuration and resolution -----
-
-    // Sidecar embedding config path: <base_dir>/lance/<db_id>/<dataset>.lance.embedding.json
-    fn dataset_embedding_config_path(&self, dataset: &str) -> std::path::PathBuf {
-        let mut base = self.lance_data_path();
-        // Ensure parent dir exists
-        if !base.exists() {
-            let _ = std::fs::create_dir_all(&base);
-        }
-        base.push(format!("{}.lance.embedding.json", dataset));
-        base
-    }
-
-    /// Persist per-dataset embedding config as JSON sidecar.
-    pub fn set_dataset_embedding_config(&self, dataset: &str, cfg: &EmbeddingConfig) -> Result<(), DBError> {
-        if self.selected_db == 0 {
-            return Err(DBError("Lance not available on admin DB 0".to_string()));
-        }
-        let p = self.dataset_embedding_config_path(dataset);
-        let data = serde_json::to_vec_pretty(cfg)
-            .map_err(|e| DBError(format!("Failed to serialize embedding config: {}", e)))?;
-        std::fs::write(&p, data)
-            .map_err(|e| DBError(format!("Failed to write embedding config {}: {}", p.display(), e)))?;
-        // Invalidate embedder cache entry for this dataset
-        {
-            let mut map = self.embedders.write().unwrap();
-            map.remove(&(self.selected_db, dataset.to_string()));
-        }
-        {
-            let mut map_img = self.image_embedders.write().unwrap();
-            map_img.remove(&(self.selected_db, dataset.to_string()));
-        }
-        Ok(())
-    }
-
-    /// Load per-dataset embedding config.
-    pub fn get_dataset_embedding_config(&self, dataset: &str) -> Result<EmbeddingConfig, DBError> {
-        if self.selected_db == 0 {
-            return Err(DBError("Lance not available on admin DB 0".to_string()));
-        }
-        let p = self.dataset_embedding_config_path(dataset);
-        if !p.exists() {
-            return Err(DBError(format!(
-                "Embedding config not set for dataset '{}'. Use LANCE.EMBEDDING CONFIG SET ... or RPC to configure.",
-                dataset
-            )));
-        }
-        let data = std::fs::read(&p)
-            .map_err(|e| DBError(format!("Failed to read embedding config {}: {}", p.display(), e)))?;
-        let cfg: EmbeddingConfig = serde_json::from_slice(&data)
-            .map_err(|e| DBError(format!("Failed to parse embedding config {}: {}", p.display(), e)))?;
-        Ok(cfg)
-    }
-
-    /// Resolve or build an embedder for (db_id, dataset). Caches instance.
-    pub fn get_embedder_for(&self, dataset: &str) -> Result<Arc<dyn Embedder>, DBError> {
-        if self.selected_db == 0 {
-            return Err(DBError("Lance not available on admin DB 0".to_string()));
-        }
-        // Fast path
-        {
-            let map = self.embedders.read().unwrap();
-            if let Some(e) = map.get(&(self.selected_db, dataset.to_string())) {
-                return Ok(e.clone());
-            }
-        }
-        // Load config and instantiate
-        let cfg = self.get_dataset_embedding_config(dataset)?;
-        let emb = create_embedder(&cfg)?;
-        {
-            let mut map = self.embedders.write().unwrap();
-            map.insert((self.selected_db, dataset.to_string()), emb.clone());
-        }
-        Ok(emb)
-    }
-
-    /// Resolve or build an IMAGE embedder for (db_id, dataset). Caches instance.
-    pub fn get_image_embedder_for(&self, dataset: &str) -> Result<Arc<dyn ImageEmbedder>, DBError> {
-        if self.selected_db == 0 {
-            return Err(DBError("Lance not available on admin DB 0".to_string()));
-        }
-        // Fast path
-        {
-            let map = self.image_embedders.read().unwrap();
-            if let Some(e) = map.get(&(self.selected_db, dataset.to_string())) {
-                return Ok(e.clone());
-            }
-        }
-        // Load config and instantiate
-        let cfg = self.get_dataset_embedding_config(dataset)?;
-        let emb = create_image_embedder(&cfg)?;
-        {
-            let mut map = self.image_embedders.write().unwrap();
-            map.insert((self.selected_db, dataset.to_string()), emb.clone());
-        }
-        Ok(emb)
-    }
-
-    /// Download image bytes from a URI with safety checks (size, timeout, content-type, optional host allowlist).
-    /// Env overrides:
-    /// - HERODB_IMAGE_MAX_BYTES (u64, default 10485760)
-    /// - HERODB_IMAGE_FETCH_TIMEOUT_SECS (u64, default 30)
-    /// - HERODB_IMAGE_ALLOWED_HOSTS (comma-separated, optional)
-    pub fn fetch_image_bytes_from_uri(&self, uri: &str) -> Result<Vec<u8>, DBError> {
-        // Basic scheme validation
-        if !(uri.starts_with("http://") || uri.starts_with("https://")) {
-            return Err(DBError("Only http(s) URIs are supported for image fetch".into()));
-        }
-        // Parse host (naive) for allowlist check
-        let host = {
-            let after_scheme = match uri.find("://") {
-                Some(i) => &uri[i + 3..],
-                None => uri,
-            };
-            let end = after_scheme.find('/').unwrap_or(after_scheme.len());
-            let host_port = &after_scheme[..end];
-            host_port.split('@').last().unwrap_or(host_port).split(':').next().unwrap_or(host_port).to_string()
-        };
-
-        let max_bytes: u64 = std::env::var("HERODB_IMAGE_MAX_BYTES").ok().and_then(|s| s.parse::<u64>().ok()).unwrap_or(10 * 1024 * 1024);
-        let timeout_secs: u64 = std::env::var("HERODB_IMAGE_FETCH_TIMEOUT_SECS").ok().and_then(|s| s.parse::<u64>().ok()).unwrap_or(30);
-        let allowed_hosts_env = std::env::var("HERODB_IMAGE_ALLOWED_HOSTS").ok();
-        if let Some(allow) = allowed_hosts_env {
-            if !allow.split(',').map(|s| s.trim()).filter(|s| !s.is_empty()).any(|h| h.eq_ignore_ascii_case(&host)) {
-                return Err(DBError(format!("Host '{}' not allowed for image fetch (HERODB_IMAGE_ALLOWED_HOSTS)", host)));
-            }
-        }
-
-        let agent: Agent = AgentBuilder::new()
-            .timeout_read(Duration::from_secs(timeout_secs))
-            .timeout_write(Duration::from_secs(timeout_secs))
-            .build();
-
-        let resp = agent.get(uri).call().map_err(|e| DBError(format!("HTTP GET failed: {}", e)))?;
-        // Validate content-type
-        let ctype = resp.header("Content-Type").unwrap_or("");
-        let ctype_main = ctype.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
-        if !ctype_main.starts_with("image/") {
-            return Err(DBError(format!("Remote content-type '{}' is not image/*", ctype)));
-        }
-
-        // Read with cap
-        let mut reader = resp.into_reader();
-        let mut buf: Vec<u8> = Vec::with_capacity(8192);
-        let mut tmp = [0u8; 8192];
-        let mut total: u64 = 0;
-        loop {
-            let n = reader.read(&mut tmp).map_err(|e| DBError(format!("Read error: {}", e)))?;
-            if n == 0 { break; }
-            total += n as u64;
-            if total > max_bytes {
-                return Err(DBError(format!("Image exceeds max allowed bytes {}", max_bytes)));
-            }
-            buf.extend_from_slice(&tmp[..n]);
-        }
-        Ok(buf)
-    }
-
-    /// Check if current permissions allow read operations
-    pub fn has_read_permission(&self) -> bool {
-        // No DB selected -> no permissions
-        if self.selected_db == NO_DB_SELECTED {
-            return false;
-        }
-        // If an explicit permission is set for this connection, honor it.
-        if let Some(perms) = self.current_permissions.as_ref() {
-            return matches!(*perms, crate::rpc::Permissions::Read | crate::rpc::Permissions::ReadWrite);
-        }
-        // Fallback ONLY when no explicit permission context (e.g., JSON-RPC flows without SELECT).
-        match crate::admin_meta::verify_access(
-            &self.option.dir,
-            self.option.backend.clone(),
-            &self.option.admin_secret,
-            self.selected_db,
-            None,
-        ) {
-            Ok(Some(crate::rpc::Permissions::Read)) | Ok(Some(crate::rpc::Permissions::ReadWrite)) => true,
-            _ => false,
-        }
-    }
-
-    /// Check if current permissions allow write operations
-    pub fn has_write_permission(&self) -> bool {
-        // No DB selected -> no permissions
-        if self.selected_db == NO_DB_SELECTED {
-            return false;
-        }
-        // If an explicit permission is set for this connection, honor it.
-        if let Some(perms) = self.current_permissions.as_ref() {
-            return matches!(*perms, crate::rpc::Permissions::ReadWrite);
-        }
-        // Fallback ONLY when no explicit permission context (e.g., JSON-RPC flows without SELECT).
-        match crate::admin_meta::verify_access(
-            &self.option.dir,
-            self.option.backend.clone(),
-            &self.option.admin_secret,
-            self.selected_db,
-            None,
-        ) {
-            Ok(Some(crate::rpc::Permissions::ReadWrite)) => true,
-            _ => false,
-        }
+    
+    fn should_encrypt_db(&self, db_index: u64) -> bool {
+        // DB 0-9 are non-encrypted, DB 10+ are encrypted
+        self.option.encrypt && db_index >= 10
     }
 
     // ----- BLPOP waiter helpers -----

src/sym.rs

@@ -1,123 +0,0 @@
-//! sym.rs — Stateless symmetric encryption (Phase 1)
-//!
-//! Commands implemented (RESP):
-//! - SYM KEYGEN
-//! - SYM ENCRYPT <key_b64> <message>
-//! - SYM DECRYPT <key_b64> <ciphertext_b64>
-//!
-//! Notes:
-//! - Raw key: exactly 32 bytes, provided as Base64 in commands.
-//! - Cipher: XChaCha20-Poly1305 (AEAD) without AAD in Phase 1
-//! - Ciphertext binary layout: [version:1][nonce:24][ciphertext||tag]
-//! - Encoding for wire I/O: Base64
-
-use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
-use chacha20poly1305::{
-    aead::{Aead, KeyInit, OsRng},
-    XChaCha20Poly1305, XNonce,
-};
-use rand::RngCore;
-
-use crate::protocol::Protocol;
-
-const VERSION: u8 = 1;
-const NONCE_LEN: usize = 24;
-const TAG_LEN: usize = 16;
-
-#[derive(Debug)]
-pub enum SymWireError {
-    InvalidKey,
-    BadEncoding,
-    BadFormat,
-    BadVersion(u8),
-    Crypto,
-}
-
-impl SymWireError {
-    fn to_protocol(self) -> Protocol {
-        match self {
-            SymWireError::InvalidKey => Protocol::err("ERR sym: invalid key"),
-            SymWireError::BadEncoding => Protocol::err("ERR sym: bad encoding"),
-            SymWireError::BadFormat => Protocol::err("ERR sym: bad format"),
-            SymWireError::BadVersion(v) => Protocol::err(&format!("ERR sym: unsupported version {}", v)),
-            SymWireError::Crypto => Protocol::err("ERR sym: auth failed"),
-        }
-    }
-}
-
-fn decode_key_b64(s: &str) -> Result<chacha20poly1305::Key, SymWireError> {
-    let bytes = B64.decode(s.as_bytes()).map_err(|_| SymWireError::BadEncoding)?;
-    if bytes.len() != 32 {
-        return Err(SymWireError::InvalidKey);
-    }
-    Ok(chacha20poly1305::Key::from_slice(&bytes).to_owned())
-}
-
-fn encrypt_blob(key: &chacha20poly1305::Key, plaintext: &[u8]) -> Result<Vec<u8>, SymWireError> {
-    let cipher = XChaCha20Poly1305::new(key);
-
-    let mut nonce_bytes = [0u8; NONCE_LEN];
-    OsRng.fill_bytes(&mut nonce_bytes);
-    let nonce = XNonce::from_slice(&nonce_bytes);
-
-    let mut out = Vec::with_capacity(1 + NONCE_LEN + plaintext.len() + TAG_LEN);
-    out.push(VERSION);
-    out.extend_from_slice(&nonce_bytes);
-
-    let ct = cipher.encrypt(nonce, plaintext).map_err(|_| SymWireError::Crypto)?;
-    out.extend_from_slice(&ct);
-    Ok(out)
-}
-
-fn decrypt_blob(key: &chacha20poly1305::Key, blob: &[u8]) -> Result<Vec<u8>, SymWireError> {
-    if blob.len() < 1 + NONCE_LEN + TAG_LEN {
-        return Err(SymWireError::BadFormat);
-    }
-    let ver = blob[0];
-    if ver != VERSION {
-        return Err(SymWireError::BadVersion(ver));
-    }
-    let nonce = XNonce::from_slice(&blob[1..1 + NONCE_LEN]);
-    let ct = &blob[1 + NONCE_LEN..];
-
-    let cipher = XChaCha20Poly1305::new(key);
-    cipher.decrypt(nonce, ct).map_err(|_| SymWireError::Crypto)
-}
-
-// ---------- Command handlers (RESP) ----------
-
-pub async fn cmd_sym_keygen() -> Protocol {
-    let mut key_bytes = [0u8; 32];
-    OsRng.fill_bytes(&mut key_bytes);
-    let key_b64 = B64.encode(key_bytes);
-    Protocol::BulkString(key_b64)
-}
-
-pub async fn cmd_sym_encrypt(key_b64: &str, message: &str) -> Protocol {
-    let key = match decode_key_b64(key_b64) {
-        Ok(k) => k,
-        Err(e) => return e.to_protocol(),
-    };
-    match encrypt_blob(&key, message.as_bytes()) {
-        Ok(blob) => Protocol::BulkString(B64.encode(blob)),
-        Err(e) => e.to_protocol(),
-    }
-}
-
-pub async fn cmd_sym_decrypt(key_b64: &str, ct_b64: &str) -> Protocol {
-    let key = match decode_key_b64(key_b64) {
-        Ok(k) => k,
-        Err(e) => return e.to_protocol(),
-    };
-    let blob = match B64.decode(ct_b64.as_bytes()) {
-        Ok(b) => b,
-        Err(_) => return SymWireError::BadEncoding.to_protocol(),
-    };
-    match decrypt_blob(&key, &blob) {
-        Ok(pt) => match String::from_utf8(pt) {
-            Ok(s) => Protocol::BulkString(s),
-            Err(_) => Protocol::err("ERR sym: invalid UTF-8 plaintext"),
-        },
-        Err(e) => e.to_protocol(),
-    }
-}

src/tantivy_search.rs

@@ -1,709 +0,0 @@
-use crate::error::DBError;
-use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use std::path::PathBuf;
-use std::sync::{Arc, RwLock};
-use tantivy::{
-    collector::TopDocs,
-    directory::MmapDirectory,
-    query::{BooleanQuery, Occur, Query, QueryParser, TermQuery},
-    schema::{
-        DateOptions, Field, IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions, STORED, STRING,
-    },
-    tokenizer::TokenizerManager,
-    DateTime, Index, IndexReader, IndexWriter, TantivyDocument, Term,
-};
-use tantivy::schema::Value;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum FieldDef {
-    Text {
-        stored: bool,
-        indexed: bool,
-        tokenized: bool,
-        fast: bool,
-    },
-    Numeric {
-        stored: bool,
-        indexed: bool,
-        fast: bool,
-        precision: NumericType,
-    },
-    Tag {
-        stored: bool,
-        separator: String,
-        case_sensitive: bool,
-    },
-    Geo {
-        stored: bool,
-    },
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum NumericType {
-    I64,
-    U64,
-    F64,
-    Date,
-}
-
-pub struct IndexSchema {
-    schema: Schema,
-    fields: HashMap<String, (Field, FieldDef)>,
-    default_search_fields: Vec<Field>,
-}
-
-pub struct TantivySearch {
-    index: Index,
-    writer: Arc<RwLock<IndexWriter>>,
-    reader: IndexReader,
-    index_schema: IndexSchema,
-    name: String,
-    config: IndexConfig,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct IndexConfig {
-    pub language: String,
-    pub stopwords: Vec<String>,
-    pub stemming: bool,
-    pub max_doc_count: Option<usize>,
-    pub default_score: f64,
-}
-
-impl Default for IndexConfig {
-    fn default() -> Self {
-        IndexConfig {
-            language: "english".to_string(),
-            stopwords: vec![],
-            stemming: true,
-            max_doc_count: None,
-            default_score: 1.0,
-        }
-    }
-}
-
-impl TantivySearch {
-    pub fn new_with_schema(
-        base_path: PathBuf,
-        name: String,
-        field_definitions: Vec<(String, FieldDef)>,
-        config: Option<IndexConfig>,
-    ) -> Result<Self, DBError> {
-        let index_path = base_path.join(&name);
-        std::fs::create_dir_all(&index_path)
-            .map_err(|e| DBError(format!("Failed to create index dir: {}", e)))?;
-
-        // Build schema from field definitions
-        let mut schema_builder = Schema::builder();
-        let mut fields = HashMap::new();
-        let mut default_search_fields = Vec::new();
-
-        // Always add a document ID field
-        let id_field = schema_builder.add_text_field("_id", STRING | STORED);
-        fields.insert(
-            "_id".to_string(),
-            (
-                id_field,
-                FieldDef::Text {
-                    stored: true,
-                    indexed: true,
-                    tokenized: false,
-                    fast: false,
-                },
-            ),
-        );
-
-        // Add user-defined fields
-        for (field_name, field_def) in field_definitions {
-            let field = match &field_def {
-                FieldDef::Text {
-                    stored,
-                    indexed,
-                    tokenized,
-                    fast: _fast,
-                } => {
-                    let mut text_options = TextOptions::default();
-                    if *stored {
-                        text_options = text_options.set_stored();
-                    }
-                    if *indexed {
-                        let indexing_options = if *tokenized {
-                            TextFieldIndexing::default()
-                                .set_tokenizer("default")
-                                .set_index_option(IndexRecordOption::WithFreqsAndPositions)
-                        } else {
-                            TextFieldIndexing::default()
-                                .set_tokenizer("raw")
-                                .set_index_option(IndexRecordOption::Basic)
-                        };
-                        text_options = text_options.set_indexing_options(indexing_options);
-                        let f = schema_builder.add_text_field(&field_name, text_options);
-                        if *tokenized {
-                            default_search_fields.push(f);
-                        }
-                        f
-                    } else {
-                        schema_builder.add_text_field(&field_name, text_options)
-                    }
-                }
-                FieldDef::Numeric {
-                    stored,
-                    indexed,
-                    fast,
-                    precision,
-                } => match precision {
-                    NumericType::I64 => {
-                        let mut opts = NumericOptions::default();
-                        if *stored {
-                            opts = opts.set_stored();
-                        }
-                        if *indexed {
-                            opts = opts.set_indexed();
-                        }
-                        if *fast {
-                            opts = opts.set_fast();
-                        }
-                        schema_builder.add_i64_field(&field_name, opts)
-                    }
-                    NumericType::U64 => {
-                        let mut opts = NumericOptions::default();
-                        if *stored {
-                            opts = opts.set_stored();
-                        }
-                        if *indexed {
-                            opts = opts.set_indexed();
-                        }
-                        if *fast {
-                            opts = opts.set_fast();
-                        }
-                        schema_builder.add_u64_field(&field_name, opts)
-                    }
-                    NumericType::F64 => {
-                        let mut opts = NumericOptions::default();
-                        if *stored {
-                            opts = opts.set_stored();
-                        }
-                        if *indexed {
-                            opts = opts.set_indexed();
-                        }
-                        if *fast {
-                            opts = opts.set_fast();
-                        }
-                        schema_builder.add_f64_field(&field_name, opts)
-                    }
-                    NumericType::Date => {
-                        let mut opts = DateOptions::default();
-                        if *stored {
-                            opts = opts.set_stored();
-                        }
-                        if *indexed {
-                            opts = opts.set_indexed();
-                        }
-                        if *fast {
-                            opts = opts.set_fast();
-                        }
-                        schema_builder.add_date_field(&field_name, opts)
-                    }
-                },
-                FieldDef::Tag {
-                    stored,
-                    separator: _,
-                    case_sensitive: _,
-                } => {
-                    let mut text_options = TextOptions::default();
-                    if *stored {
-                        text_options = text_options.set_stored();
-                    }
-                    text_options = text_options.set_indexing_options(
-                        TextFieldIndexing::default()
-                            .set_tokenizer("raw")
-                            .set_index_option(IndexRecordOption::Basic),
-                    );
-                    schema_builder.add_text_field(&field_name, text_options)
-                }
-                FieldDef::Geo { stored } => {
-                    // For now, store as two f64 fields for lat/lon
-                    let mut opts = NumericOptions::default();
-                    if *stored {
-                        opts = opts.set_stored();
-                    }
-                    opts = opts.set_indexed().set_fast();
-                    let lat_field =
-                        schema_builder.add_f64_field(&format!("{}_lat", field_name), opts.clone());
-                    let lon_field =
-                        schema_builder.add_f64_field(&format!("{}_lon", field_name), opts);
-                    fields.insert(
-                        format!("{}_lat", field_name),
-                        (
-                            lat_field,
-                            FieldDef::Numeric {
-                                stored: *stored,
-                                indexed: true,
-                                fast: true,
-                                precision: NumericType::F64,
-                            },
-                        ),
-                    );
-                    fields.insert(
-                        format!("{}_lon", field_name),
-                        (
-                            lon_field,
-                            FieldDef::Numeric {
-                                stored: *stored,
-                                indexed: true,
-                                fast: true,
-                                precision: NumericType::F64,
-                            },
-                        ),
-                    );
-                    continue; // Skip adding the geo field itself
-                }
-            };
-            fields.insert(field_name.clone(), (field, field_def));
-        }
-
-        let schema = schema_builder.build();
-        let index_schema = IndexSchema {
-            schema: schema.clone(),
-            fields,
-            default_search_fields,
-        };
-
-        // Create or open index
-        let dir = MmapDirectory::open(&index_path)
-            .map_err(|e| DBError(format!("Failed to open index directory: {}", e)))?;
-        let mut index =
-            Index::open_or_create(dir, schema).map_err(|e| DBError(format!("Failed to create index: {}", e)))?;
-
-        // Configure tokenizers
-        let tokenizer_manager = TokenizerManager::default();
-        index.set_tokenizers(tokenizer_manager);
-
-        let writer = index
-            .writer(15_000_000)
-            .map_err(|e| DBError(format!("Failed to create index writer: {}", e)))?;
-        let reader = index
-            .reader()
-            .map_err(|e| DBError(format!("Failed to create reader: {}", e)))?;
-
-        let config = config.unwrap_or_default();
-
-        Ok(TantivySearch {
-            index,
-            writer: Arc::new(RwLock::new(writer)),
-            reader,
-            index_schema,
-            name,
-            config,
-        })
-    }
-
-    pub fn add_document_with_fields(
-        &self,
-        doc_id: &str,
-        fields: HashMap<String, String>,
-    ) -> Result<(), DBError> {
-        let mut writer = self
-            .writer
-            .write()
-            .map_err(|e| DBError(format!("Failed to acquire writer lock: {}", e)))?;
-
-        // Delete existing document with same ID
-        if let Some((id_field, _)) = self.index_schema.fields.get("_id") {
-            writer.delete_term(Term::from_field_text(*id_field, doc_id));
-        }
-
-        // Create new document
-        let mut doc = tantivy::doc!();
-
-        // Add document ID
-        if let Some((id_field, _)) = self.index_schema.fields.get("_id") {
-            doc.add_text(*id_field, doc_id);
-        }
-
-        // Add other fields based on schema
-        for (field_name, field_value) in fields {
-            if let Some((field, field_def)) = self.index_schema.fields.get(&field_name) {
-                match field_def {
-                    FieldDef::Text { .. } => {
-                        doc.add_text(*field, &field_value);
-                    }
-                    FieldDef::Numeric { precision, .. } => match precision {
-                        NumericType::I64 => {
-                            if let Ok(v) = field_value.parse::<i64>() {
-                                doc.add_i64(*field, v);
-                            }
-                        }
-                        NumericType::U64 => {
-                            if let Ok(v) = field_value.parse::<u64>() {
-                                doc.add_u64(*field, v);
-                            }
-                        }
-                        NumericType::F64 => {
-                            if let Ok(v) = field_value.parse::<f64>() {
-                                doc.add_f64(*field, v);
-                            }
-                        }
-                        NumericType::Date => {
-                            if let Ok(v) = field_value.parse::<i64>() {
-                                doc.add_date(*field, DateTime::from_timestamp_millis(v));
-                            }
-                        }
-                    },
-                    FieldDef::Tag {
-                        separator,
-                        case_sensitive,
-                        ..
-                    } => {
-                        let tags = if !case_sensitive {
-                            field_value.to_lowercase()
-                        } else {
-                            field_value.clone()
-                        };
-                        for tag in tags.split(separator.as_str()) {
-                            doc.add_text(*field, tag.trim());
-                        }
-                    }
-                    FieldDef::Geo { .. } => {
-                        let parts: Vec<&str> = field_value.split(',').collect();
-                        if parts.len() == 2 {
-                            if let (Ok(lat), Ok(lon)) =
-                                (parts[0].parse::<f64>(), parts[1].parse::<f64>())
-                            {
-                                if let Some((lat_field, _)) =
-                                    self.index_schema.fields.get(&format!("{}_lat", field_name))
-                                {
-                                    doc.add_f64(*lat_field, lat);
-                                }
-                                if let Some((lon_field, _)) =
-                                    self.index_schema.fields.get(&format!("{}_lon", field_name))
-                                {
-                                    doc.add_f64(*lon_field, lon);
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        writer
-            .add_document(doc)
-            .map_err(|e| DBError(format!("Failed to add document: {}", e)))?;
-        writer
-            .commit()
-            .map_err(|e| DBError(format!("Failed to commit: {}", e)))?;
-        // Make new documents visible to searches
-        self.reader
-            .reload()
-            .map_err(|e| DBError(format!("Failed to reload reader: {}", e)))?;
-        Ok(())
-    }
-
-    pub fn search_with_options(
-        &self,
-        query_str: &str,
-        options: SearchOptions,
-    ) -> Result<SearchResults, DBError> {
-        // Ensure reader is up to date with latest commits
-        self.reader
-            .reload()
-            .map_err(|e| DBError(format!("Failed to reload reader: {}", e)))?;
-        let searcher = self.reader.searcher();
-
-        // Ensure we have searchable fields
-        if self.index_schema.default_search_fields.is_empty() {
-            return Err(DBError("No searchable fields defined in schema".to_string()));
-        }
-
-        // Parse query based on search fields
-        let query_parser = QueryParser::for_index(
-            &self.index,
-            self.index_schema.default_search_fields.clone(),
-        );
-        let parsed_query = query_parser
-            .parse_query(query_str)
-            .map_err(|e| DBError(format!("Failed to parse query: {}", e)))?;
-        let mut clauses: Vec<(Occur, Box<dyn Query>)> = vec![(Occur::Must, parsed_query)];
-
-        // Apply filters if any
-        for filter in options.filters {
-            if let Some((field, field_def)) = self.index_schema.fields.get(&filter.field) {
-                match filter.filter_type {
-                    FilterType::Equals(value) => {
-                        match field_def {
-                            FieldDef::Text { .. } | FieldDef::Tag { .. } => {
-                                let term_query =
-                                    TermQuery::new(Term::from_field_text(*field, &value), IndexRecordOption::Basic);
-                                clauses.push((Occur::Must, Box::new(term_query)));
-                            }
-                            FieldDef::Numeric { precision, .. } => {
-                                // Equals on numeric fields: parse to the right numeric type and use term query
-                                match precision {
-                                    NumericType::I64 => {
-                                        if let Ok(v) = value.parse::<i64>() {
-                                            let term = Term::from_field_i64(*field, v);
-                                            let tq = TermQuery::new(term, IndexRecordOption::Basic);
-                                            clauses.push((Occur::Must, Box::new(tq)));
-                                        }
-                                    }
-                                    NumericType::U64 => {
-                                        if let Ok(v) = value.parse::<u64>() {
-                                            let term = Term::from_field_u64(*field, v);
-                                            let tq = TermQuery::new(term, IndexRecordOption::Basic);
-                                            clauses.push((Occur::Must, Box::new(tq)));
-                                        }
-                                    }
-                                    NumericType::F64 => {
-                                        if let Ok(v) = value.parse::<f64>() {
-                                            let term = Term::from_field_f64(*field, v);
-                                            let tq = TermQuery::new(term, IndexRecordOption::Basic);
-                                            clauses.push((Occur::Must, Box::new(tq)));
-                                        }
-                                    }
-                                    NumericType::Date => {
-                                        if let Ok(v) = value.parse::<i64>() {
-                                            let dt = DateTime::from_timestamp_millis(v);
-                                            let term = Term::from_field_date(*field, dt);
-                                            let tq = TermQuery::new(term, IndexRecordOption::Basic);
-                                            clauses.push((Occur::Must, Box::new(tq)));
-                                        }
-                                    }
-                                }
-                            }
-                            FieldDef::Geo { .. } => {
-                                // Geo equals isn't supported in this simplified version
-                            }
-                        }
-                    }
-                    FilterType::Range { .. } => {
-                        // TODO: Implement numeric range queries by building a RangeQuery per type
-                    }
-                    FilterType::InSet(values) => {
-                        // OR across values
-                        let mut sub_clauses: Vec<(Occur, Box<dyn Query>)> = vec![];
-                        for value in values {
-                            let term_query = TermQuery::new(
-                                Term::from_field_text(*field, &value),
-                                IndexRecordOption::Basic,
-                            );
-                            sub_clauses.push((Occur::Should, Box::new(term_query)));
-                        }
-                        clauses.push((Occur::Must, Box::new(BooleanQuery::new(sub_clauses))));
-                    }
-                }
-            }
-        }
-
-        let final_query: Box<dyn Query> = if clauses.len() == 1 {
-            clauses.pop().unwrap().1
-        } else {
-            Box::new(BooleanQuery::new(clauses))
-        };
-
-        // Execute search
-        let top_docs = searcher
-            .search(&*final_query, &TopDocs::with_limit(options.limit + options.offset))
-            .map_err(|e| DBError(format!("Search failed: {}", e)))?;
-        let total_hits = top_docs.len();
-        let mut documents = Vec::new();
-
-        for (score, doc_address) in top_docs.into_iter().skip(options.offset).take(options.limit) {
-            let retrieved_doc: TantivyDocument = searcher
-                .doc(doc_address)
-                .map_err(|e| DBError(format!("Failed to retrieve doc: {}", e)))?;
-
-            let mut doc_fields = HashMap::new();
-
-            // Extract stored fields (or synthesize)
-            for (field_name, (field, field_def)) in &self.index_schema.fields {
-                match field_def {
-                    FieldDef::Text { stored, .. } | FieldDef::Tag { stored, .. } => {
-                        if *stored {
-                            if let Some(value) = retrieved_doc.get_first(*field) {
-                                if let Some(text) = value.as_str() {
-                                    doc_fields.insert(field_name.clone(), text.to_string());
-                                }
-                            }
-                        }
-                    }
-                    FieldDef::Numeric {
-                        stored, precision, ..
-                    } => {
-                        if *stored {
-                            let value_str = match precision {
-                                NumericType::I64 => retrieved_doc
-                                    .get_first(*field)
-                                    .and_then(|v| v.as_i64())
-                                    .map(|v| v.to_string()),
-                                NumericType::U64 => retrieved_doc
-                                    .get_first(*field)
-                                    .and_then(|v| v.as_u64())
-                                    .map(|v| v.to_string()),
-                                NumericType::F64 => retrieved_doc
-                                    .get_first(*field)
-                                    .and_then(|v| v.as_f64())
-                                    .map(|v| v.to_string()),
-                                NumericType::Date => retrieved_doc
-                                    .get_first(*field)
-                                    .and_then(|v| v.as_datetime())
-                                    .map(|v| v.into_timestamp_millis().to_string()),
-                            };
-                            if let Some(v) = value_str {
-                                doc_fields.insert(field_name.clone(), v);
-                            }
-                        }
-                    }
-                    FieldDef::Geo { stored } => {
-                        if *stored {
-                            let lat_field = self
-                                .index_schema
-                                .fields
-                                .get(&format!("{}_lat", field_name))
-                                .unwrap()
-                                .0;
-                            let lon_field = self
-                                .index_schema
-                                .fields
-                                .get(&format!("{}_lon", field_name))
-                                .unwrap()
-                                .0;
-                            let lat = retrieved_doc.get_first(lat_field).and_then(|v| v.as_f64());
-                            let lon = retrieved_doc.get_first(lon_field).and_then(|v| v.as_f64());
-                            if let (Some(lat), Some(lon)) = (lat, lon) {
-                                doc_fields.insert(field_name.clone(), format!("{},{}", lat, lon));
-                            }
-                        }
-                    }
-                }
-            }
-
-            documents.push(SearchDocument {
-                fields: doc_fields,
-                score,
-            });
-        }
-
-        Ok(SearchResults {
-            total: total_hits,
-            documents,
-        })
-    }
-
-    pub fn get_info(&self) -> Result<IndexInfo, DBError> {
-        let searcher = self.reader.searcher();
-        let num_docs = searcher.num_docs();
-        let fields_info: Vec<FieldInfo> = self
-            .index_schema
-            .fields
-            .iter()
-            .map(|(name, (_, def))| FieldInfo {
-                name: name.clone(),
-                field_type: format!("{:?}", def),
-            })
-            .collect();
-        Ok(IndexInfo {
-            name: self.name.clone(),
-            num_docs,
-            fields: fields_info,
-            config: self.config.clone(),
-        })
-    }
-
-    /// Delete a document by its _id term. Returns true if the document existed before deletion.
-    pub fn delete_document_by_id(&self, doc_id: &str) -> Result<bool, DBError> {
-        // Determine existence by running a tiny term query
-        let existed = if let Some((id_field, _)) = self.index_schema.fields.get("_id") {
-            let term = Term::from_field_text(*id_field, doc_id);
-            let searcher = self.reader.searcher();
-            let tq = TermQuery::new(term.clone(), IndexRecordOption::Basic);
-            let hits = searcher
-                .search(&tq, &TopDocs::with_limit(1))
-                .map_err(|e| DBError(format!("Failed to search for existing doc: {}", e)))?;
-            !hits.is_empty()
-        } else {
-            false
-        };
-
-        // Perform deletion and commit
-        let mut writer = self
-            .writer
-            .write()
-            .map_err(|e| DBError(format!("Failed to acquire writer lock: {}", e)))?;
-        if let Some((id_field, _)) = self.index_schema.fields.get("_id") {
-            writer.delete_term(Term::from_field_text(*id_field, doc_id));
-        }
-        writer
-            .commit()
-            .map_err(|e| DBError(format!("Failed to commit delete: {}", e)))?;
-        // Refresh reader to observe deletion
-        self.reader
-            .reload()
-            .map_err(|e| DBError(format!("Failed to reload reader: {}", e)))?;
-
-        Ok(existed)
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct SearchOptions {
-    pub limit: usize,
-    pub offset: usize,
-    pub filters: Vec<Filter>,
-    pub sort_by: Option<String>,
-    pub return_fields: Option<Vec<String>>,
-    pub highlight: bool,
-}
-
-impl Default for SearchOptions {
-    fn default() -> Self {
-        SearchOptions {
-            limit: 10,
-            offset: 0,
-            filters: vec![],
-            sort_by: None,
-            return_fields: None,
-            highlight: false,
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct Filter {
-    pub field: String,
-    pub filter_type: FilterType,
-}
-
-#[derive(Debug, Clone)]
-pub enum FilterType {
-    Equals(String),
-    Range { min: String, max: String },
-    InSet(Vec<String>),
-}
-
-#[derive(Debug)]
-pub struct SearchResults {
-    pub total: usize,
-    pub documents: Vec<SearchDocument>,
-}
-
-#[derive(Debug)]
-pub struct SearchDocument {
-    pub fields: HashMap<String, String>,
-    pub score: f32,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct IndexInfo {
-    pub name: String,
-    pub num_docs: u64,
-    pub fields: Vec<FieldInfo>,
-    pub config: IndexConfig,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct FieldInfo {
-    pub name: String,
-    pub field_type: String,
-}

tests/debug_hset.rs

@@ -1,5 +1,4 @@
 use herodb::{server::Server, options::DBOption};
-use std::path::PathBuf;
 use std::time::Duration;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::TcpStream;
@@ -23,13 +22,12 @@ async fn debug_hset_simple() {
     
     let port = 16500;
     let option = DBOption {
-        dir: PathBuf::from(test_dir),
+        dir: test_dir.to_string(),
         port,
         debug: false,
         encrypt: false,
         encryption_key: None,
         backend: herodb::options::BackendType::Redb,
-        admin_secret: "test-admin".to_string(),
     };
     
     let mut server = Server::new(option).await;
@@ -50,12 +48,6 @@ async fn debug_hset_simple() {
     sleep(Duration::from_millis(200)).await;
     
     let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)).await.unwrap();
-    // Acquire ReadWrite permissions on this connection
-    let resp = send_command(
-        &mut stream,
-        "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n",
-    ).await;
-    assert!(resp.contains("OK"), "Failed SELECT handshake: {}", resp);
     
     // Test simple HSET
     println!("Testing HSET...");

tests/debug_hset_simple.rs

@@ -1,5 +1,4 @@
 use herodb::{server::Server, options::DBOption};
-use std::path::PathBuf;
 use std::time::Duration;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::TcpStream;
@@ -14,13 +13,12 @@ async fn debug_hset_return_value() {
     std::fs::create_dir_all(&test_dir).unwrap();
     
     let option = DBOption {
-        dir: PathBuf::from(test_dir),
+        dir: test_dir.to_string(),
         port: 16390,
         debug: false,
         encrypt: false,
         encryption_key: None,
         backend: herodb::options::BackendType::Redb,
-        admin_secret: "test-admin".to_string(),
     };
     
     let mut server = Server::new(option).await;
@@ -42,19 +40,12 @@ async fn debug_hset_return_value() {
     
     // Connect and test HSET
     let mut stream = TcpStream::connect("127.0.0.1:16390").await.unwrap();
-
-    // Acquire ReadWrite permissions for this new connection
-    let handshake = "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n";
-    stream.write_all(handshake.as_bytes()).await.unwrap();
-    let mut buffer = [0; 1024];
-    let n = stream.read(&mut buffer).await.unwrap();
-    let resp = String::from_utf8_lossy(&buffer[..n]);
-    assert!(resp.contains("OK"), "Failed SELECT handshake: {}", resp);
     
     // Send HSET command
     let cmd = "*4\r\n$4\r\nHSET\r\n$4\r\nhash\r\n$6\r\nfield1\r\n$6\r\nvalue1\r\n";
     stream.write_all(cmd.as_bytes()).await.unwrap();
     
+    let mut buffer = [0; 1024];
     let n = stream.read(&mut buffer).await.unwrap();
     let response = String::from_utf8_lossy(&buffer[..n]);
     

tests/lance_integration_tests.rs

@@ -1,484 +0,0 @@
-use redis::{Client, Connection, RedisResult, Value};
-use std::process::{Child, Command};
-use std::time::Duration;
-
-use jsonrpsee::http_client::{HttpClient, HttpClientBuilder};
-use herodb::rpc::{BackendType, DatabaseConfig, RpcClient};
-use base64::Engine;
-use tokio::time::sleep;
-
-// ------------------------
-// Helpers
-// ------------------------
-
-fn get_redis_connection(port: u16) -> Connection {
-    let connection_info = format!("redis://127.0.0.1:{}", port);
-    let client = Client::open(connection_info).unwrap();
-    let mut attempts = 0;
-    loop {
-        match client.get_connection() {
-            Ok(mut conn) => {
-                if redis::cmd("PING").query::<String>(&mut conn).is_ok() {
-                    return conn;
-                }
-            }
-            Err(e) => {
-                if attempts >= 3600 {
-                    panic!("Failed to connect to Redis server after 3600 attempts: {}", e);
-                }
-            }
-        }
-        attempts += 1;
-        std::thread::sleep(Duration::from_millis(500));
-    }
-}
-
-async fn get_rpc_client(port: u16) -> HttpClient {
-    let url = format!("http://127.0.0.1:{}", port + 1); // RPC port = Redis port + 1
-    HttpClientBuilder::default().build(url).unwrap()
-}
-
-/// Wait until RPC server is responsive (getServerStats succeeds) or panic after retries.
-async fn wait_for_rpc_ready(client: &HttpClient, max_attempts: u32, delay: Duration) {
-    for _ in 0..max_attempts {
-        match client.get_server_stats().await {
-            Ok(_) => return,
-            Err(_) => {
-                sleep(delay).await;
-            }
-        }
-    }
-    panic!("RPC server did not become ready in time");
-}
-
-// A guard to ensure the server process is killed when it goes out of scope and test dir cleaned.
-struct ServerProcessGuard {
-    process: Child,
-    test_dir: String,
-}
-
-impl Drop for ServerProcessGuard {
-    fn drop(&mut self) {
-        eprintln!("Killing server process (pid: {})...", self.process.id());
-        if let Err(e) = self.process.kill() {
-            eprintln!("Failed to kill server process: {}", e);
-        }
-        match self.process.wait() {
-            Ok(status) => eprintln!("Server process exited with: {}", status),
-            Err(e) => eprintln!("Failed to wait on server process: {}", e),
-        }
-
-        // Clean up the specific test directory
-        eprintln!("Cleaning up test directory: {}", self.test_dir);
-        if let Err(e) = std::fs::remove_dir_all(&self.test_dir) {
-            eprintln!("Failed to clean up test directory: {}", e);
-        }
-    }
-}
-
-// Helper to set up the server and return guard + ports
-async fn setup_server() -> (ServerProcessGuard, u16) {
-    use std::sync::atomic::{AtomicU16, Ordering};
-    static PORT_COUNTER: AtomicU16 = AtomicU16::new(17500);
-    let port = PORT_COUNTER.fetch_add(1, Ordering::SeqCst);
-
-    let test_dir = format!("/tmp/herodb_lance_test_{}", port);
-
-    // Clean up previous test data
-    if std::path::Path::new(&test_dir).exists() {
-        let _ = std::fs::remove_dir_all(&test_dir);
-    }
-    std::fs::create_dir_all(&test_dir).unwrap();
-
-    // Start the server in a subprocess with RPC enabled (follows tantivy test pattern)
-    let child = Command::new("cargo")
-        .args(&[
-            "run",
-            "--",
-            "--dir",
-            &test_dir,
-            "--port",
-            &port.to_string(),
-            "--rpc-port",
-            &(port + 1).to_string(),
-            "--enable-rpc",
-            "--debug",
-            "--admin-secret",
-            "test-admin",
-        ])
-        .spawn()
-        .expect("Failed to start server process");
-
-    let guard = ServerProcessGuard {
-        process: child,
-        test_dir,
-    };
-
-    // Give the server time to build and start (cargo run may compile first)
-    // Increase significantly to accommodate first-time dependency compilation in CI.
-    std::thread::sleep(Duration::from_millis(60000));
-
-    (guard, port)
-}
-
-// Convenient helpers for assertions on redis::Value
-fn value_is_ok(v: &Value) -> bool {
-    match v {
-        Value::Okay => true,
-        Value::Status(s) if s == "OK" => true,
-        Value::Data(d) if d == b"OK" => true,
-        _ => false,
-    }
-}
-
-fn value_is_int_eq(v: &Value, expected: i64) -> bool {
-    matches!(v, Value::Int(n) if *n == expected)
-}
-
-fn value_is_str_eq(v: &Value, expected: &str) -> bool {
-    match v {
-        Value::Status(s) => s == expected,
-        Value::Data(d) => String::from_utf8_lossy(d) == expected,
-        _ => false,
-    }
-}
-
-fn to_string_lossy(v: &Value) -> String {
-    match v {
-        Value::Nil => "Nil".to_string(),
-        Value::Int(n) => n.to_string(),
-        Value::Status(s) => s.clone(),
-        Value::Okay => "OK".to_string(),
-        Value::Data(d) => String::from_utf8_lossy(d).to_string(),
-        Value::Bulk(items) => {
-            let inner: Vec<String> = items.iter().map(to_string_lossy).collect();
-            format!("[{}]", inner.join(", "))
-        }
-    }
-}
-
-// Extract ids from LANCE.SEARCH / LANCE.SEARCHIMAGE reply which is:
-// Array of elements: [ [id, score, [k,v,...]], [id, score, ...], ... ]
-fn extract_hit_ids(v: &Value) -> Vec<String> {
-    let mut ids = Vec::new();
-    if let Value::Bulk(items) = v {
-        for item in items {
-            if let Value::Bulk(row) = item {
-                if !row.is_empty() {
-                    // first element is id (Data or Status)
-                    let id = match &row[0] {
-                        Value::Data(d) => String::from_utf8_lossy(d).to_string(),
-                        Value::Status(s) => s.clone(),
-                        Value::Int(n) => n.to_string(),
-                        _ => continue,
-                    };
-                    ids.push(id);
-                }
-            }
-        }
-    }
-    ids
-}
-
-// Check whether a Bulk array (RESP array) contains a given string element.
-fn bulk_contains_string(v: &Value, needle: &str) -> bool {
-    match v {
-        Value::Bulk(items) => items.iter().any(|it| match it {
-            Value::Data(d) => String::from_utf8_lossy(d).contains(needle),
-            Value::Status(s) => s.contains(needle),
-            Value::Bulk(_) => bulk_contains_string(it, needle),
-            _ => false,
-        }),
-        _ => false,
-    }
-}
-
-// ------------------------
-// Test: Lance end-to-end (RESP) using only local embedders
-// ------------------------
-
-#[tokio::test]
-async fn test_lance_end_to_end() {
-    let (_guard, port) = setup_server().await;
-
-    // First, wait for RESP to be available; this also gives cargo-run child ample time to finish building.
-    // Reuse the helper that retries PING until success.
-    {
-        let _conn_ready = get_redis_connection(port);
-        // Drop immediately; we only needed readiness.
-    }
-
-    // Build RPC client and create a Lance DB
-    let rpc_client = get_rpc_client(port).await;
-    // Ensure RPC server is listening before we issue createDatabase (allow longer warm-up to accommodate first-build costs)
-    wait_for_rpc_ready(&rpc_client, 3600, Duration::from_millis(250)).await;
-
-    let db_config = DatabaseConfig {
-        name: Some("media-db".to_string()),
-        storage_path: None,
-        max_size: None,
-        redis_version: None,
-    };
-
-    let db_id = rpc_client
-        .create_database(BackendType::Lance, db_config, None)
-        .await
-        .expect("create_database Lance failed");
-
-    assert_eq!(db_id, 1, "Expected first Lance DB id to be 1");
-
-    // Add access keys
-    let _ = rpc_client
-        .add_access_key(db_id, "readwrite_key".to_string(), "readwrite".to_string())
-        .await
-        .expect("add_access_key readwrite failed");
-
-    let _ = rpc_client
-        .add_access_key(db_id, "read_key".to_string(), "read".to_string())
-        .await
-        .expect("add_access_key read failed");
-
-    // Connect to Redis and SELECT DB with readwrite key
-    let mut conn = get_redis_connection(port);
-
-    let sel_ok: RedisResult<String> = redis::cmd("SELECT")
-        .arg(db_id)
-        .arg("KEY")
-        .arg("readwrite_key")
-        .query(&mut conn);
-    assert!(sel_ok.is_ok(), "SELECT db with key failed: {:?}", sel_ok);
-    assert_eq!(sel_ok.unwrap(), "OK");
-
-    // 1) Configure embedding providers: textset -> testhash dim 64, imageset -> testimagehash dim 512
-    let v = redis::cmd("LANCE.EMBEDDING")
-        .arg("CONFIG")
-        .arg("SET")
-        .arg("textset")
-        .arg("PROVIDER")
-        .arg("testhash")
-        .arg("MODEL")
-        .arg("any")
-        .arg("PARAM")
-        .arg("dim")
-        .arg("64")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "Embedding config set (text) not OK: {}", to_string_lossy(&v));
-
-    let v = redis::cmd("LANCE.EMBEDDING")
-        .arg("CONFIG")
-        .arg("SET")
-        .arg("imageset")
-        .arg("PROVIDER")
-        .arg("testimagehash")
-        .arg("MODEL")
-        .arg("any")
-        .arg("PARAM")
-        .arg("dim")
-        .arg("512")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "Embedding config set (image) not OK: {}", to_string_lossy(&v));
-
-    // 2) Create datasets
-    let v = redis::cmd("LANCE.CREATE")
-        .arg("textset")
-        .arg("DIM")
-        .arg(64)
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "LANCE.CREATE textset failed: {}", to_string_lossy(&v));
-
-    let v = redis::cmd("LANCE.CREATE")
-        .arg("imageset")
-        .arg("DIM")
-        .arg(512)
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "LANCE.CREATE imageset failed: {}", to_string_lossy(&v));
-
-    // 3) Store two text documents
-    let v = redis::cmd("LANCE.STORE")
-        .arg("textset")
-        .arg("ID")
-        .arg("doc-1")
-        .arg("TEXT")
-        .arg("The quick brown fox jumps over the lazy dog")
-        .arg("META")
-        .arg("title")
-        .arg("Fox")
-        .arg("category")
-        .arg("animal")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "LANCE.STORE doc-1 failed: {}", to_string_lossy(&v));
-
-    let v = redis::cmd("LANCE.STORE")
-        .arg("textset")
-        .arg("ID")
-        .arg("doc-2")
-        .arg("TEXT")
-        .arg("A fast auburn fox vaulted a sleepy canine")
-        .arg("META")
-        .arg("title")
-        .arg("Paraphrase")
-        .arg("category")
-        .arg("animal")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "LANCE.STORE doc-2 failed: {}", to_string_lossy(&v));
-
-    // 4) Store two images via BYTES (local fake bytes; embedder only hashes bytes, not decoding)
-    let img1: Vec<u8> = b"local-image-bytes-1-abcdefghijklmnopqrstuvwxyz".to_vec();
-    let img2: Vec<u8> = b"local-image-bytes-2-ABCDEFGHIJKLMNOPQRSTUVWXYZ".to_vec();
-    let img1_b64 = base64::engine::general_purpose::STANDARD.encode(&img1);
-    let img2_b64 = base64::engine::general_purpose::STANDARD.encode(&img2);
-
-    let v = redis::cmd("LANCE.STOREIMAGE")
-        .arg("imageset")
-        .arg("ID")
-        .arg("img-1")
-        .arg("BYTES")
-        .arg(&img1_b64)
-        .arg("META")
-        .arg("title")
-        .arg("Local1")
-        .arg("group")
-        .arg("demo")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "LANCE.STOREIMAGE img-1 failed: {}", to_string_lossy(&v));
-
-    let v = redis::cmd("LANCE.STOREIMAGE")
-        .arg("imageset")
-        .arg("ID")
-        .arg("img-2")
-        .arg("BYTES")
-        .arg(&img2_b64)
-        .arg("META")
-        .arg("title")
-        .arg("Local2")
-        .arg("group")
-        .arg("demo")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "LANCE.STOREIMAGE img-2 failed: {}", to_string_lossy(&v));
-
-    // 5) Search text: K 2 QUERY "quick brown fox" RETURN 1 title
-    let v = redis::cmd("LANCE.SEARCH")
-        .arg("textset")
-        .arg("K")
-        .arg(2)
-        .arg("QUERY")
-        .arg("quick brown fox")
-        .arg("RETURN")
-        .arg(1)
-        .arg("title")
-        .query::<Value>(&mut conn)
-        .unwrap();
-
-    // Should be an array of hits
-    let ids = extract_hit_ids(&v);
-    assert!(
-        ids.contains(&"doc-1".to_string()) || ids.contains(&"doc-2".to_string()),
-        "LANCE.SEARCH should return doc-1/doc-2; got: {}",
-        to_string_lossy(&v)
-    );
-
-    // With FILTER on category
-    let v = redis::cmd("LANCE.SEARCH")
-        .arg("textset")
-        .arg("K")
-        .arg(2)
-        .arg("QUERY")
-        .arg("fox jumps")
-        .arg("FILTER")
-        .arg("category = 'animal'")
-        .arg("RETURN")
-        .arg(1)
-        .arg("title")
-        .query::<Value>(&mut conn)
-        .unwrap();
-
-    let ids_f = extract_hit_ids(&v);
-    assert!(
-        !ids_f.is_empty(),
-        "Filtered LANCE.SEARCH should return at least one document; got: {}",
-        to_string_lossy(&v)
-    );
-
-    // 6) Search images with QUERYBYTES
-    let query_img: Vec<u8> = b"local-image-query-3-1234567890".to_vec();
-    let query_img_b64 = base64::engine::general_purpose::STANDARD.encode(&query_img);
-
-    let v = redis::cmd("LANCE.SEARCHIMAGE")
-        .arg("imageset")
-        .arg("K")
-        .arg(2)
-        .arg("QUERYBYTES")
-        .arg(&query_img_b64)
-        .arg("RETURN")
-        .arg(1)
-        .arg("title")
-        .query::<Value>(&mut conn)
-        .unwrap();
-
-    // Should get 2 hits (img-1 and img-2) in some order; assert array non-empty
-    let img_ids = extract_hit_ids(&v);
-    assert!(
-        !img_ids.is_empty(),
-        "LANCE.SEARCHIMAGE should return non-empty results; got: {}",
-        to_string_lossy(&v)
-    );
-
-    // 7) Inspect datasets
-    let v = redis::cmd("LANCE.LIST").query::<Value>(&mut conn).unwrap();
-    assert!(
-        bulk_contains_string(&v, "textset"),
-        "LANCE.LIST missing textset: {}",
-        to_string_lossy(&v)
-    );
-    assert!(
-        bulk_contains_string(&v, "imageset"),
-        "LANCE.LIST missing imageset: {}",
-        to_string_lossy(&v)
-    );
-
-    // INFO textset
-    let info_text = redis::cmd("LANCE.INFO")
-        .arg("textset")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    // INFO returns Array [k,v,k,v,...] including "dimension" "64" and "row_count" "...".
-    let info_str = to_string_lossy(&info_text);
-    assert!(
-        info_str.contains("dimension") && info_str.contains("64"),
-        "LANCE.INFO textset should include dimension 64; got: {}",
-        info_str
-    );
-
-    // 8) Delete by id and drop datasets
-    let v = redis::cmd("LANCE.DEL")
-        .arg("textset")
-        .arg("doc-2")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    // Returns SimpleString "1" or Int 1 depending on encoding path; accept either
-    assert!(
-        value_is_int_eq(&v, 1) || value_is_str_eq(&v, "1"),
-        "LANCE.DEL doc-2 expected 1; got {}",
-        to_string_lossy(&v)
-    );
-
-    let v = redis::cmd("LANCE.DROP")
-        .arg("textset")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "LANCE.DROP textset failed: {}", to_string_lossy(&v));
-
-    let v = redis::cmd("LANCE.DROP")
-        .arg("imageset")
-        .query::<Value>(&mut conn)
-        .unwrap();
-    assert!(value_is_ok(&v), "LANCE.DROP imageset failed: {}", to_string_lossy(&v));
-}

tests/redis_integration_tests.rs

@@ -12,15 +12,7 @@ fn get_redis_connection(port: u16) -> Connection {
         match client.get_connection() {
             Ok(mut conn) => {
                 if redis::cmd("PING").query::<String>(&mut conn).is_ok() {
-                    // Acquire ReadWrite permissions on this connection
-                    let sel: RedisResult<String> = redis::cmd("SELECT")
-                        .arg(0)
-                        .arg("KEY")
-                        .arg("test-admin")
-                        .query(&mut conn);
-                    if sel.is_ok() {
-                        return conn;
-                    }
+                    return conn;
                 }
             }
             Err(e) => {
@@ -86,8 +78,6 @@ fn setup_server() -> (ServerProcessGuard, u16) {
             "--port",
             &port.to_string(),
             "--debug",
-            "--admin-secret",
-            "test-admin",
         ])
         .spawn()
         .expect("Failed to start server process");

tests/redis_tests.rs

@@ -1,5 +1,4 @@
 use herodb::{server::Server, options::DBOption};
-use std::path::PathBuf;
 use std::time::Duration;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::TcpStream;
@@ -18,35 +17,24 @@ async fn start_test_server(test_name: &str) -> (Server, u16) {
     std::fs::create_dir_all(&test_dir).unwrap();
     
     let option = DBOption {
-        dir: PathBuf::from(test_dir),
+        dir: test_dir,
         port,
         debug: true,
         encrypt: false,
         encryption_key: None,
         backend: herodb::options::BackendType::Redb,
-        admin_secret: "test-admin".to_string(),
     };
     
     let server = Server::new(option).await;
     (server, port)
 }
 
- // Helper function to connect to the test server
+// Helper function to connect to the test server
 async fn connect_to_server(port: u16) -> TcpStream {
     let mut attempts = 0;
     loop {
         match TcpStream::connect(format!("127.0.0.1:{}", port)).await {
-            Ok(mut stream) => {
-                // Obtain ReadWrite permissions for this connection by selecting DB 0 with admin key
-                let resp = send_command(
-                    &mut stream,
-                    "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n",
-                ).await;
-                if !resp.contains("OK") {
-                    panic!("Failed to acquire write permissions via SELECT 0 KEY test-admin: {}", resp);
-                }
-                return stream;
-            }
+            Ok(stream) => return stream,
             Err(_) if attempts < 10 => {
                 attempts += 1;
                 sleep(Duration::from_millis(100)).await;

tests/rpc_tests.rs

@@ -1,86 +0,0 @@
-use herodb::rpc::{BackendType, DatabaseConfig};
-use herodb::admin_meta;
-use herodb::options::BackendType as OptionsBackendType;
-use std::path::Path;
-
-#[tokio::test]
-async fn test_rpc_server_basic() {
-    // This test would require starting the RPC server in a separate thread
-    // For now, we'll just test that the types compile correctly
-
-    // Test serialization of types
-    let backend = BackendType::Redb;
-    let config = DatabaseConfig {
-        name: Some("test_db".to_string()),
-        storage_path: Some("/tmp/test".to_string()),
-        max_size: Some(1024 * 1024),
-        redis_version: Some("7.0".to_string()),
-    };
-
-    let backend_json = serde_json::to_string(&backend).unwrap();
-    let config_json = serde_json::to_string(&config).unwrap();
-
-    assert_eq!(backend_json, "\"Redb\"");
-    assert!(config_json.contains("test_db"));
-}
-
-#[tokio::test]
-async fn test_database_config_serialization() {
-    let config = DatabaseConfig {
-        name: Some("my_db".to_string()),
-        storage_path: None,
-        max_size: Some(1000000),
-        redis_version: Some("7.0".to_string()),
-    };
-
-    let json = serde_json::to_value(&config).unwrap();
-    assert_eq!(json["name"], "my_db");
-    assert_eq!(json["max_size"], 1000000);
-    assert_eq!(json["redis_version"], "7.0");
-}
-
-#[tokio::test]
-async fn test_backend_type_serialization() {
-    // Test that both Redb and Sled backends serialize correctly
-    let redb_backend = BackendType::Redb;
-    let sled_backend = BackendType::Sled;
-
-    let redb_json = serde_json::to_string(&redb_backend).unwrap();
-    let sled_json = serde_json::to_string(&sled_backend).unwrap();
-
-    assert_eq!(redb_json, "\"Redb\"");
-    assert_eq!(sled_json, "\"Sled\"");
-
-    // Test deserialization
-    let redb_deserialized: BackendType = serde_json::from_str(&redb_json).unwrap();
-    let sled_deserialized: BackendType = serde_json::from_str(&sled_json).unwrap();
-
-    assert!(matches!(redb_deserialized, BackendType::Redb));
-    assert!(matches!(sled_deserialized, BackendType::Sled));
-}
-
-#[tokio::test]
-async fn test_database_name_persistence() {
-    let base_dir = "/tmp/test_db_name_persistence";
-    let admin_secret = "test-admin-secret";
-    let backend = OptionsBackendType::Redb;
-    let db_id = 1;
-    let test_name = "test-database-name";
-
-    // Clean up any existing test data
-    let _ = std::fs::remove_dir_all(base_dir);
-
-    // Set the database name
-    admin_meta::set_database_name(Path::new(base_dir), backend.clone(), admin_secret, db_id, test_name)
-        .expect("Failed to set database name");
-
-    // Retrieve the database name
-    let retrieved_name = admin_meta::get_database_name(Path::new(base_dir), backend, admin_secret, db_id)
-        .expect("Failed to get database name");
-
-    // Verify the name matches
-    assert_eq!(retrieved_name, Some(test_name.to_string()));
-
-    // Clean up
-    let _ = std::fs::remove_dir_all(base_dir);
-}

tests/simple_integration_test.rs

@@ -1,5 +1,4 @@
 use herodb::{server::Server, options::DBOption};
-use std::path::PathBuf;
 use std::time::Duration;
 use tokio::time::sleep;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
@@ -20,13 +19,12 @@ async fn start_test_server(test_name: &str) -> (Server, u16) {
     std::fs::create_dir_all(&test_dir).unwrap();
     
     let option = DBOption {
-        dir: PathBuf::from(test_dir),
+        dir: test_dir,
         port,
         debug: true,
         encrypt: false,
         encryption_key: None,
         backend: herodb::options::BackendType::Redb,
-        admin_secret: "test-admin".to_string(),
     };
     
     let server = Server::new(option).await;
@@ -36,16 +34,9 @@ async fn start_test_server(test_name: &str) -> (Server, u16) {
 // Helper function to send Redis command and get response
 async fn send_redis_command(port: u16, command: &str) -> String {
     let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)).await.unwrap();
-    
-    // Acquire ReadWrite permissions on this new connection
-    let handshake = "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n";
-    stream.write_all(handshake.as_bytes()).await.unwrap();
-    let mut buffer = [0; 1024];
-    let _ = stream.read(&mut buffer).await.unwrap(); // Read and ignore the OK for handshake
-    
-    // Now send the intended command
     stream.write_all(command.as_bytes()).await.unwrap();
     
+    let mut buffer = [0; 1024];
     let n = stream.read(&mut buffer).await.unwrap();
     String::from_utf8_lossy(&buffer[..n]).to_string()
 }
@@ -193,19 +184,12 @@ async fn test_transaction_operations() {
     
     sleep(Duration::from_millis(100)).await;
     
-     // Use a single connection for the transaction
+    // Use a single connection for the transaction
     let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)).await.unwrap();
     
-    // Acquire write permissions for this connection
-    let handshake = "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n";
-    stream.write_all(handshake.as_bytes()).await.unwrap();
-    let mut buffer = [0; 1024];
-    let n = stream.read(&mut buffer).await.unwrap();
-    let resp = String::from_utf8_lossy(&buffer[..n]);
-    assert!(resp.contains("OK"));
-    
     // Test MULTI
     stream.write_all("*1\r\n$5\r\nMULTI\r\n".as_bytes()).await.unwrap();
+    let mut buffer = [0; 1024];
     let n = stream.read(&mut buffer).await.unwrap();
     let response = String::from_utf8_lossy(&buffer[..n]);
     assert!(response.contains("OK"));

tests/simple_redis_test.rs

@@ -1,5 +1,4 @@
 use herodb::{server::Server, options::DBOption};
-use std::path::PathBuf;
 use std::time::Duration;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::TcpStream;
@@ -18,13 +17,12 @@ async fn start_test_server(test_name: &str) -> (Server, u16) {
     std::fs::create_dir_all(&test_dir).unwrap();
     
     let option = DBOption {
-        dir: PathBuf::from(test_dir),
+        dir: test_dir,
         port,
         debug: false,
         encrypt: false,
         encryption_key: None,
         backend: herodb::options::BackendType::Redb,
-        admin_secret: "test-admin".to_string(),
     };
     
     let server = Server::new(option).await;
@@ -40,22 +38,12 @@ async fn send_command(stream: &mut TcpStream, command: &str) -> String {
     String::from_utf8_lossy(&buffer[..n]).to_string()
 }
 
- // Helper function to connect to the test server
+// Helper function to connect to the test server
 async fn connect_to_server(port: u16) -> TcpStream {
     let mut attempts = 0;
     loop {
         match TcpStream::connect(format!("127.0.0.1:{}", port)).await {
-            Ok(mut stream) => {
-                // Acquire ReadWrite permissions for this connection
-                let resp = send_command(
-                    &mut stream,
-                    "*4\r\n$6\r\nSELECT\r\n$1\r\n0\r\n$3\r\nKEY\r\n$10\r\ntest-admin\r\n",
-                ).await;
-                if !resp.contains("OK") {
-                    panic!("Failed to acquire write permissions via SELECT 0 KEY test-admin: {}", resp);
-                }
-                return stream;
-            }
+            Ok(stream) => return stream,
             Err(_) if attempts < 10 => {
                 attempts += 1;
                 sleep(Duration::from_millis(100)).await;
@@ -109,21 +97,14 @@ async fn test_hset_clean_db() {
     sleep(Duration::from_millis(200)).await;
     
     let mut stream = connect_to_server(port).await;
-
-    // Ensure clean DB state (admin DB 0 may be shared due to global singleton)
-    let flush = send_command(&mut stream, "*1\r\n$7\r\nFLUSHDB\r\n").await;
-    assert!(flush.contains("OK"), "Failed to FLUSHDB: {}", flush);
-
-    // Test HSET - should return 1 for new field (use a unique key name to avoid collisions)
-    let key = "hash_clean";
-    let hset_cmd = format!("*4\r\n$4\r\nHSET\r\n${}\r\n{}\r\n$6\r\nfield1\r\n$6\r\nvalue1\r\n", key.len(), key);
-    let response = send_command(&mut stream, &hset_cmd).await;
+    
+    // Test HSET - should return 1 for new field
+    let response = send_command(&mut stream, "*4\r\n$4\r\nHSET\r\n$4\r\nhash\r\n$6\r\nfield1\r\n$6\r\nvalue1\r\n").await;
     println!("HSET response: {}", response);
     assert!(response.contains("1"), "Expected HSET to return 1, got: {}", response);
     
     // Test HGET
-    let hget_cmd = format!("*3\r\n$4\r\nHGET\r\n${}\r\n{}\r\n$6\r\nfield1\r\n", key.len(), key);
-    let response = send_command(&mut stream, &hget_cmd).await;
+    let response = send_command(&mut stream, "*3\r\n$4\r\nHGET\r\n$4\r\nhash\r\n$6\r\nfield1\r\n").await;
     println!("HGET response: {}", response);
     assert!(response.contains("value1"));
 }

tests/tantivy_integration_tests.rs

@@ -1,294 +0,0 @@
-use redis::{Client, Connection, RedisResult};
-use std::process::{Child, Command};
-use std::time::Duration;
-use jsonrpsee::http_client::{HttpClientBuilder, HttpClient};
-use herodb::rpc::{RpcClient, BackendType, DatabaseConfig};
-
-// Helper function to get Redis connection, retrying until successful
-fn get_redis_connection(port: u16) -> Connection {
-    let connection_info = format!("redis://127.0.0.1:{}", port);
-    let client = Client::open(connection_info).unwrap();
-    let mut attempts = 0;
-    loop {
-        match client.get_connection() {
-            Ok(mut conn) => {
-                if redis::cmd("PING").query::<String>(&mut conn).is_ok() {
-                    return conn;
-                }
-            }
-            Err(e) => {
-                if attempts >= 120 {
-                    panic!(
-                        "Failed to connect to Redis server after 120 attempts: {}",
-                        e
-                    );
-                }
-            }
-        }
-        attempts += 1;
-        std::thread::sleep(Duration::from_millis(100));
-    }
-}
-
-// Helper function to get RPC client
-async fn get_rpc_client(port: u16) -> HttpClient {
-    let url = format!("http://127.0.0.1:{}", port + 1); // RPC port is Redis port + 1
-    let client = HttpClientBuilder::default().build(url).unwrap();
-    client
-}
-
-// A guard to ensure the server process is killed when it goes out of scope
-struct ServerProcessGuard {
-    process: Child,
-    test_dir: String,
-}
-
-impl Drop for ServerProcessGuard {
-    fn drop(&mut self) {
-        println!("Killing server process (pid: {})...", self.process.id());
-        if let Err(e) = self.process.kill() {
-            eprintln!("Failed to kill server process: {}", e);
-        }
-        match self.process.wait() {
-            Ok(status) => println!("Server process exited with: {}", status),
-            Err(e) => eprintln!("Failed to wait on server process: {}", e),
-        }
-
-        // Clean up the specific test directory
-        println!("Cleaning up test directory: {}", self.test_dir);
-        if let Err(e) = std::fs::remove_dir_all(&self.test_dir) {
-            eprintln!("Failed to clean up test directory: {}", e);
-        }
-    }
-}
-
-// Helper to set up the server and return connections
-async fn setup_server() -> (ServerProcessGuard, u16, Connection, HttpClient) {
-    use std::sync::atomic::{AtomicU16, Ordering};
-    static PORT_COUNTER: AtomicU16 = AtomicU16::new(16500);
-    let port = PORT_COUNTER.fetch_add(1, Ordering::SeqCst);
-
-    let test_dir = format!("/tmp/herodb_tantivy_test_{}", port);
-
-    // Clean up previous test data
-    if std::path::Path::new(&test_dir).exists() {
-        let _ = std::fs::remove_dir_all(&test_dir);
-    }
-    std::fs::create_dir_all(&test_dir).unwrap();
-
-    // Start the server in a subprocess
-    let child = Command::new("cargo")
-        .args(&[
-            "run",
-            "--",
-            "--dir",
-            &test_dir,
-            "--port",
-            &port.to_string(),
-            "--rpc-port",
-            &(port + 1).to_string(),
-            "--enable-rpc",
-            "--debug",
-            "--admin-secret",
-            "test-admin",
-        ])
-        .spawn()
-        .expect("Failed to start server process");
-
-    // Create a new guard that also owns the test directory path
-    let guard = ServerProcessGuard {
-        process: child,
-        test_dir,
-    };
-
-    // Give the server time to build and start (cargo run may compile first)
-    std::thread::sleep(Duration::from_millis(3000));
-
-    let conn = get_redis_connection(port);
-    let rpc_client = get_rpc_client(port).await;
-
-    (guard, port, conn, rpc_client)
-}
-
-
-#[tokio::test]
-async fn test_tantivy_full_text_search() {
-    let (_server_guard, _port, mut conn, rpc_client) = setup_server().await;
-
-    // Create a Tantivy database via RPC
-    let db_config = DatabaseConfig {
-        name: Some("test_tantivy_db".to_string()),
-        storage_path: None,
-        max_size: None,
-        redis_version: None,
-    };
-
-    let db_id = rpc_client.create_database(BackendType::Tantivy, db_config, None).await.unwrap();
-    assert_eq!(db_id, 1);
-
-    // Add readwrite access key
-    let _ = rpc_client.add_access_key(db_id, "readwrite_key".to_string(), "readwrite".to_string()).await.unwrap();
-
-    // Add read-only access key
-    let _ = rpc_client.add_access_key(db_id, "read_key".to_string(), "read".to_string()).await.unwrap();
-
-    // Test with readwrite permissions
-    test_tantivy_with_readwrite_permissions(&mut conn, db_id).await;
-
-    // Test with read-only permissions
-    test_tantivy_with_read_permissions(&mut conn, db_id).await;
-
-    // Test access denied for invalid key
-    test_tantivy_access_denied(&mut conn, db_id).await;
-}
-
-async fn test_tantivy_with_readwrite_permissions(conn: &mut Connection, db_id: u64) {
-    // Select database with readwrite key
-    let result: RedisResult<String> = redis::cmd("SELECT")
-        .arg(db_id)
-        .arg("KEY")
-        .arg("readwrite_key")
-        .query(conn);
-    assert!(result.is_ok());
-    assert_eq!(result.unwrap(), "OK");
-
-    // Test FT.CREATE
-    let result: RedisResult<String> = redis::cmd("FT.CREATE")
-        .arg("test_index")
-        .arg("SCHEMA")
-        .arg("title")
-        .arg("TEXT")
-        .arg("content")
-        .arg("TEXT")
-        .arg("tags")
-        .arg("TAG")
-        .query(conn);
-    assert!(result.is_ok());
-    assert_eq!(result.unwrap(), "OK");
-
-    // Test FT.ADD
-    let result: RedisResult<String> = redis::cmd("FT.ADD")
-        .arg("test_index")
-        .arg("doc1")
-        .arg("1.0")
-        .arg("title")
-        .arg("Hello World")
-        .arg("content")
-        .arg("This is a test document")
-        .arg("tags")
-        .arg("test,example")
-        .query(conn);
-    assert!(result.is_ok());
-    assert_eq!(result.unwrap(), "OK");
-
-    // Add another document
-    let result: RedisResult<String> = redis::cmd("FT.ADD")
-        .arg("test_index")
-        .arg("doc2")
-        .arg("1.0")
-        .arg("title")
-        .arg("Goodbye World")
-        .arg("content")
-        .arg("Another test document")
-        .arg("tags")
-        .arg("test,another")
-        .query(conn);
-    assert!(result.is_ok());
-    assert_eq!(result.unwrap(), "OK");
-
-    // Test FT.SEARCH
-    let result: RedisResult<Vec<String>> = redis::cmd("FT.SEARCH")
-        .arg("test_index")
-        .arg("test")
-        .query(conn);
-    assert!(result.is_ok());
-    let results = result.unwrap();
-    assert!(results.len() >= 3); // At least total count + 2 documents
-    assert_eq!(results[0], "2"); // Total matches
-
-    // Test FT.INFO
-    let result: RedisResult<Vec<String>> = redis::cmd("FT.INFO")
-        .arg("test_index")
-        .query(conn);
-    assert!(result.is_ok());
-    let info = result.unwrap();
-    assert!(info.contains(&"index_name".to_string()));
-    assert!(info.contains(&"test_index".to_string()));
-
-    // Test FT.DEL
-    let result: RedisResult<String> = redis::cmd("FT.DEL")
-        .arg("test_index")
-        .arg("doc1")
-        .query(conn);
-    assert!(result.is_ok());
-    assert_eq!(result.unwrap(), "1");
-
-    // Verify document was deleted
-    let result: RedisResult<Vec<String>> = redis::cmd("FT.SEARCH")
-        .arg("test_index")
-        .arg("Hello")
-        .query(conn);
-    assert!(result.is_ok());
-    let results = result.unwrap();
-    assert_eq!(results[0], "0"); // No matches
-
-    // Test FT.DROP
-    let result: RedisResult<String> = redis::cmd("FT.DROP")
-        .arg("test_index")
-        .query(conn);
-    assert!(result.is_ok());
-    assert_eq!(result.unwrap(), "OK");
-
-    // Verify index was dropped
-    let result: RedisResult<String> = redis::cmd("FT.INFO")
-        .arg("test_index")
-        .query(conn);
-    assert!(result.is_err()); // Should fail
-}
-
-async fn test_tantivy_with_read_permissions(conn: &mut Connection, db_id: u64) {
-    // Select database with read-only key
-    let result: RedisResult<String> = redis::cmd("SELECT")
-        .arg(db_id)
-        .arg("KEY")
-        .arg("read_key")
-        .query(conn);
-    assert!(result.is_ok());
-    assert_eq!(result.unwrap(), "OK");
-
-    // Recreate index for testing
-    let result: RedisResult<String> = redis::cmd("FT.CREATE")
-        .arg("test_index_read")
-        .arg("SCHEMA")
-        .arg("title")
-        .arg("TEXT")
-        .query(conn);
-    assert!(result.is_err()); // Should fail due to read-only permissions
-    assert!(result.unwrap_err().to_string().contains("write permission denied"));
-
-    // Add document should fail
-    let result: RedisResult<String> = redis::cmd("FT.ADD")
-        .arg("test_index_read")
-        .arg("doc1")
-        .arg("1.0")
-        .arg("title")
-        .arg("Test")
-        .query(conn);
-    assert!(result.is_err());
-    assert!(result.unwrap_err().to_string().contains("write permission denied"));
-
-    // But search should work (if index exists)
-    // First create index with write permissions, then switch to read
-    // For this test, we'll assume the index doesn't exist, so search fails differently
-}
-
-async fn test_tantivy_access_denied(conn: &mut Connection, db_id: u64) {
-    // Try to select with invalid key
-    let result: RedisResult<String> = redis::cmd("SELECT")
-        .arg(db_id)
-        .arg("KEY")
-        .arg("invalid_key")
-        .query(conn);
-    assert!(result.is_err());
-    assert!(result.unwrap_err().to_string().contains("invalid access key"));
-}

tests/usage_suite.rs

@@ -1,5 +1,4 @@
 use herodb::{options::DBOption, server::Server};
-use std::path::PathBuf;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::TcpStream;
 use tokio::time::{sleep, Duration};
@@ -18,13 +17,12 @@ async fn start_test_server(test_name: &str) -> (Server, u16) {
     std::fs::create_dir_all(&test_dir).unwrap();
 
     let option = DBOption {
-        dir: PathBuf::from(test_dir),
+        dir: test_dir,
         port,
         debug: false,
         encrypt: false,
         encryption_key: None,
         backend: herodb::options::BackendType::Redb,
-        admin_secret: "test-admin".to_string(),
     };
 
     let server = Server::new(option).await;
@@ -63,17 +61,7 @@ async fn connect(port: u16) -> TcpStream {
     let mut attempts = 0;
     loop {
         match TcpStream::connect(format!("127.0.0.1:{}", port)).await {
-            Ok(mut s) => {
-                // Acquire ReadWrite permissions for this connection using admin DB 0
-                let resp = send_cmd(&mut s, &["SELECT", "0", "KEY", "test-admin"]).await;
-                assert_contains(&resp, "OK", "SELECT 0 KEY test-admin handshake");
-
-                // Ensure clean slate per test on DB 0
-                let fl = send_cmd(&mut s, &["FLUSHDB"]).await;
-                assert_contains(&fl, "OK", "FLUSHDB after handshake");
-
-                return s;
-            }
+            Ok(s) => return s,
             Err(_) if attempts < 30 => {
                 attempts += 1;
                 sleep(Duration::from_millis(100)).await;
@@ -258,9 +246,9 @@ async fn test_01_connection_and_info() {
     let getname = send_cmd(&mut s, &["CLIENT", "GETNAME"]).await;
     assert_contains(&getname, "myapp", "CLIENT GETNAME");
 
-    // SELECT db (requires key on DB 0)
-    let sel = send_cmd(&mut s, &["SELECT", "0", "KEY", "test-admin"]).await;
-    assert_contains(&sel, "OK", "SELECT 0 with key");
+    // SELECT db
+    let sel = send_cmd(&mut s, &["SELECT", "0"]).await;
+    assert_contains(&sel, "OK", "SELECT 0");
 
     // QUIT should close connection after sending OK
     let quit = send_cmd(&mut s, &["QUIT"]).await;
@@ -291,11 +279,7 @@ async fn test_02_strings_and_expiry() {
 
     let ex0 = send_cmd(&mut s, &["EXISTS", "user:1"]).await;
     assert_contains(&ex0, "0", "EXISTS after DEL");
-    
-    // DEL non-existent should return 0
-    let del0 = send_cmd(&mut s, &["DEL", "user:1"]).await;
-    assert_contains(&del0, "0", "DEL user:1 when not exists -> 0");
-    
+
     // INCR behavior
     let i1 = send_cmd(&mut s, &["INCR", "count"]).await;
     assert_contains(&i1, "1", "INCR new key -> 1");
@@ -517,11 +501,11 @@ async fn test_07_age_stateless_suite() {
     let mut s = connect(port).await;
 
     // GENENC -> [recipient, identity]
-    let genenc = send_cmd(&mut s, &["AGE", "GENENC"]).await;
+    let gen = send_cmd(&mut s, &["AGE", "GENENC"]).await;
     assert!(
-        genenc.starts_with("*2\r\n$"),
+        gen.starts_with("*2\r\n$"),
         "AGE GENENC should return array [recipient, identity], got:\n{}",
-        genenc
+        gen
     );
 
     // Parse simple RESP array of two bulk strings to extract keys
@@ -536,7 +520,7 @@ async fn test_07_age_stateless_suite() {
         let ident = lines.next().unwrap_or("").to_string();
         (recip, ident)
     }
-    let (recipient, identity) = parse_two_bulk_array(&genenc);
+    let (recipient, identity) = parse_two_bulk_array(&gen);
     assert!(
         recipient.starts_with("age1") && identity.starts_with("AGE-SECRET-KEY-1"),
         "Unexpected AGE key formats.\nrecipient: {}\nidentity: {}",
@@ -607,7 +591,7 @@ async fn test_08_age_persistent_named_suite() {
 
     // AGE LIST
     let lst = send_cmd(&mut s, &["AGE", "LIST"]).await;
-    // After flattening, LIST returns a flat array of managed key names
+    assert_contains(&lst, "encpub", "AGE LIST label encpub");
     assert_contains(&lst, "app1", "AGE LIST includes app1");
 }