WIP2: implementing lancedb: created embedding abstraction, server-side per-dataset embedding config + updates RPC endpoints

src/rpc.rs

@@ -9,6 +9,7 @@ use sha2::{Digest, Sha256};
 use crate::server::Server;
 use crate::options::DBOption;
 use crate::admin_meta;
+use crate::embedding::{EmbeddingConfig, EmbeddingProvider};
 
 /// Database backend types
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -163,8 +164,8 @@ pub trait Rpc {
     #[method(name = "ftDrop")]
     async fn ft_drop(&self, db_id: u64, index_name: String) -> RpcResult<bool>;
 
-    // ----- LanceDB (Vector) RPC endpoints -----
-
+    // ----- LanceDB (Vector + Text) RPC endpoints -----
+    
     /// Create a new Lance dataset in a Lance-backed DB
     #[method(name = "lanceCreate")]
     async fn lance_create(
@@ -173,8 +174,8 @@ pub trait Rpc {
         name: String,
         dim: usize,
     ) -> RpcResult<bool>;
-
-    /// Store a vector (with id and metadata) into a Lance dataset
+    
+    /// Store a vector (with id and metadata) into a Lance dataset (deprecated; returns error)
     #[method(name = "lanceStore")]
     async fn lance_store(
         &self,
@@ -184,8 +185,8 @@ pub trait Rpc {
         vector: Vec<f32>,
         meta: Option<HashMap<String, String>>,
     ) -> RpcResult<bool>;
-
-    /// Search a Lance dataset with a query vector
+    
+    /// Search a Lance dataset with a query vector (deprecated; returns error)
     #[method(name = "lanceSearch")]
     async fn lance_search(
         &self,
@@ -196,7 +197,7 @@ pub trait Rpc {
         filter: Option<String>,
         return_fields: Option<Vec<String>>,
     ) -> RpcResult<serde_json::Value>;
-
+    
     /// Create an ANN index on a Lance dataset
     #[method(name = "lanceCreateIndex")]
     async fn lance_create_index(
@@ -206,14 +207,14 @@ pub trait Rpc {
         index_type: String,
         params: Option<HashMap<String, String>>,
     ) -> RpcResult<bool>;
-
+    
     /// List Lance datasets for a DB
     #[method(name = "lanceList")]
     async fn lance_list(
         &self,
         db_id: u64,
     ) -> RpcResult<Vec<String>>;
-
+    
     /// Get info for a Lance dataset
     #[method(name = "lanceInfo")]
     async fn lance_info(
@@ -221,7 +222,7 @@ pub trait Rpc {
         db_id: u64,
         name: String,
     ) -> RpcResult<serde_json::Value>;
-
+    
     /// Delete a record by id from a Lance dataset
     #[method(name = "lanceDel")]
     async fn lance_del(
@@ -230,7 +231,7 @@ pub trait Rpc {
         name: String,
         id: String,
     ) -> RpcResult<bool>;
-
+    
     /// Drop a Lance dataset
     #[method(name = "lanceDrop")]
     async fn lance_drop(
@@ -238,6 +239,49 @@ pub trait Rpc {
         db_id: u64,
         name: String,
     ) -> RpcResult<bool>;
+
+    // New: Text-first endpoints (no user-provided vectors)
+    /// Set per-dataset embedding configuration
+    #[method(name = "lanceSetEmbeddingConfig")]
+    async fn lance_set_embedding_config(
+        &self,
+        db_id: u64,
+        name: String,
+        provider: String,
+        model: String,
+        params: Option<HashMap<String, String>>,
+    ) -> RpcResult<bool>;
+
+    /// Get per-dataset embedding configuration
+    #[method(name = "lanceGetEmbeddingConfig")]
+    async fn lance_get_embedding_config(
+        &self,
+        db_id: u64,
+        name: String,
+    ) -> RpcResult<serde_json::Value>;
+
+    /// Store text; server will embed and store vector+text+meta
+    #[method(name = "lanceStoreText")]
+    async fn lance_store_text(
+        &self,
+        db_id: u64,
+        name: String,
+        id: String,
+        text: String,
+        meta: Option<HashMap<String, String>>,
+    ) -> RpcResult<bool>;
+
+    /// Search using a text query; server will embed then search
+    #[method(name = "lanceSearchText")]
+    async fn lance_search_text(
+        &self,
+        db_id: u64,
+        name: String,
+        text: String,
+        k: usize,
+        filter: Option<String>,
+        return_fields: Option<Vec<String>>,
+    ) -> RpcResult<serde_json::Value>;
 }
 
 /// RPC Server implementation
@@ -789,62 +833,33 @@ impl RpcServer for RpcServerImpl {
 
     async fn lance_store(
         &self,
-        db_id: u64,
-        name: String,
-        id: String,
-        vector: Vec<f32>,
-        meta: Option<HashMap<String, String>>,
+        _db_id: u64,
+        _name: String,
+        _id: String,
+        _vector: Vec<f32>,
+        _meta: Option<HashMap<String, String>>,
     ) -> RpcResult<bool> {
-        let server = self.get_or_create_server(db_id).await?;
-        if db_id == 0 {
-            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "Lance not allowed on DB 0", None::<()>));
-        }
-        if !matches!(server.option.backend, crate::options::BackendType::Lance) {
-            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "DB backend is not Lance", None::<()>));
-        }
-        if !server.has_write_permission() {
-            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "write permission denied", None::<()>));
-        }
-        server.lance_store()
-            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?
-            .store_vector(&name, &id, vector, meta.unwrap_or_default()).await
-            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
-        Ok(true)
+        Err(jsonrpsee::types::ErrorObjectOwned::owned(
+            -32000,
+            "Vector endpoint removed. Use lanceStoreText instead.",
+            None::<()>
+        ))
     }
 
     async fn lance_search(
         &self,
-        db_id: u64,
-        name: String,
-        vector: Vec<f32>,
-        k: usize,
-        filter: Option<String>,
-        return_fields: Option<Vec<String>>,
+        _db_id: u64,
+        _name: String,
+        _vector: Vec<f32>,
+        _k: usize,
+        _filter: Option<String>,
+        _return_fields: Option<Vec<String>>,
     ) -> RpcResult<serde_json::Value> {
-        let server = self.get_or_create_server(db_id).await?;
-        if db_id == 0 {
-            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "Lance not allowed on DB 0", None::<()>));
-        }
-        if !matches!(server.option.backend, crate::options::BackendType::Lance) {
-            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "DB backend is not Lance", None::<()>));
-        }
-        if !server.has_read_permission() {
-            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "read permission denied", None::<()>));
-        }
-        let results = server.lance_store()
-            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?
-            .search_vectors(&name, vector, k, filter, return_fields).await
-            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
-
-        let json_results: Vec<serde_json::Value> = results.into_iter().map(|(id, score, meta)| {
-            serde_json::json!({
-                "id": id,
-                "score": score,
-                "meta": meta,
-            })
-        }).collect();
-
-        Ok(serde_json::json!({ "results": json_results }))
+        Err(jsonrpsee::types::ErrorObjectOwned::owned(
+            -32000,
+            "Vector endpoint removed. Use lanceSearchText instead.",
+            None::<()>
+        ))
     }
 
     async fn lance_create_index(
@@ -958,4 +973,137 @@ impl RpcServer for RpcServerImpl {
             .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
         Ok(ok)
     }
+
+    // ----- New text-first Lance RPC implementations -----
+
+    async fn lance_set_embedding_config(
+        &self,
+        db_id: u64,
+        name: String,
+        provider: String,
+        model: String,
+        params: Option<HashMap<String, String>>,
+    ) -> RpcResult<bool> {
+        let server = self.get_or_create_server(db_id).await?;
+        if db_id == 0 {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "Lance not allowed on DB 0", None::<()>));
+        }
+        if !matches!(server.option.backend, crate::options::BackendType::Lance) {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "DB backend is not Lance", None::<()>));
+        }
+        if !server.has_write_permission() {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "write permission denied", None::<()>));
+        }
+        let prov = match provider.to_lowercase().as_str() {
+            "test-hash" | "testhash" => EmbeddingProvider::TestHash,
+            "fastembed" | "lancefastembed" => EmbeddingProvider::LanceFastEmbed,
+            "openai" | "lanceopenai" => EmbeddingProvider::LanceOpenAI,
+            other => EmbeddingProvider::LanceOther(other.to_string()),
+        };
+        let cfg = EmbeddingConfig {
+            provider: prov,
+            model,
+            params: params.unwrap_or_default(),
+        };
+        server.set_dataset_embedding_config(&name, &cfg)
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
+        Ok(true)
+    }
+
+    async fn lance_get_embedding_config(
+        &self,
+        db_id: u64,
+        name: String,
+    ) -> RpcResult<serde_json::Value> {
+        let server = self.get_or_create_server(db_id).await?;
+        if db_id == 0 {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "Lance not allowed on DB 0", None::<()>));
+        }
+        if !matches!(server.option.backend, crate::options::BackendType::Lance) {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "DB backend is not Lance", None::<()>));
+        }
+        if !server.has_read_permission() {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "read permission denied", None::<()>));
+        }
+        let cfg = server.get_dataset_embedding_config(&name)
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
+        Ok(serde_json::json!({
+            "provider": match cfg.provider {
+                EmbeddingProvider::TestHash => "test-hash",
+                EmbeddingProvider::LanceFastEmbed => "lancefastembed",
+                EmbeddingProvider::LanceOpenAI => "lanceopenai",
+                EmbeddingProvider::LanceOther(ref s) => s,
+            },
+            "model": cfg.model,
+            "params": cfg.params
+        }))
+    }
+
+    async fn lance_store_text(
+        &self,
+        db_id: u64,
+        name: String,
+        id: String,
+        text: String,
+        meta: Option<HashMap<String, String>>,
+    ) -> RpcResult<bool> {
+        let server = self.get_or_create_server(db_id).await?;
+        if db_id == 0 {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "Lance not allowed on DB 0", None::<()>));
+        }
+        if !matches!(server.option.backend, crate::options::BackendType::Lance) {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "DB backend is not Lance", None::<()>));
+        }
+        if !server.has_write_permission() {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "write permission denied", None::<()>));
+        }
+        let embedder = server.get_embedder_for(&name)
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
+        let vector = embedder.embed(&text)
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
+        server.lance_store()
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?
+            .store_vector(&name, &id, vector, meta.unwrap_or_default(), Some(text)).await
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
+        Ok(true)
+    }
+
+    async fn lance_search_text(
+        &self,
+        db_id: u64,
+        name: String,
+        text: String,
+        k: usize,
+        filter: Option<String>,
+        return_fields: Option<Vec<String>>,
+    ) -> RpcResult<serde_json::Value> {
+        let server = self.get_or_create_server(db_id).await?;
+        if db_id == 0 {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "Lance not allowed on DB 0", None::<()>));
+        }
+        if !matches!(server.option.backend, crate::options::BackendType::Lance) {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "DB backend is not Lance", None::<()>));
+        }
+        if !server.has_read_permission() {
+            return Err(jsonrpsee::types::ErrorObjectOwned::owned(-32000, "read permission denied", None::<()>));
+        }
+        let embedder = server.get_embedder_for(&name)
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
+        let qv = embedder.embed(&text)
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
+        let results = server.lance_store()
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?
+            .search_vectors(&name, qv, k, filter, return_fields).await
+            .map_err(|e| jsonrpsee::types::ErrorObjectOwned::owned(-32000, e.0, None::<()>))?;
+
+        let json_results: Vec<serde_json::Value> = results.into_iter().map(|(id, score, meta)| {
+            serde_json::json!({
+                "id": id,
+                "score": score,
+                "meta": meta,
+            })
+        }).collect();
+
+        Ok(serde_json::json!({ "results": json_results }))
+    }
 }