feat: Add index management and scroll functionality to Qdrant client

- Add support for creating and deleting indexes in Qdrant collections.
- Implement scrolling functionality for retrieving points in batches.
- Enhance point retrieval with options for including payload and vector.
- Add comprehensive error handling for all new operations.
- Introduce new structures for parameters and responses.
This commit is contained in:
Mahmoud Emad
2025-03-26 13:34:07 +02:00
parent 228abe36a3
commit cf98822749
4 changed files with 814 additions and 0 deletions

View File

@@ -224,3 +224,87 @@ pub fn (mut self QDrantClient) is_collection_exists(params CollectionExistencePa
return json.decode(QDrantResponse[CollectionExistenceResponse], response.data)! return json.decode(QDrantResponse[CollectionExistenceResponse], response.data)!
} }
// Parameters for creating an index
@[params]
pub struct CreateIndexParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
field_name string @[json: 'field_name'; required] // Name of the field to create index for
field_schema FieldSchema @[json: 'field_schema'; required] // Schema of the field
wait ?bool @[json: 'wait'] // Whether to wait until the changes have been applied
}
// Field schema for index
pub struct FieldSchema {
pub mut:
field_type string @[json: 'type'; required] // Type of the field (keyword, integer, float, geo)
}
// Response structure for index operations
pub struct IndexOperationResponse {
pub mut:
status string @[json: 'status']
operation_id int @[json: 'operation_id']
}
// Create an index for a field in a collection
pub fn (mut self QDrantClient) create_index(params CreateIndexParams) !QDrantResponse[IndexOperationResponse] {
mut http_conn := self.httpclient()!
mut data := {
'field_name': params.field_name
'field_schema': json.encode(params.field_schema)
}
if params.wait != none {
data['wait'] = params.wait.str()
}
req := httpconnection.Request{
method: .put
prefix: '/collections/${params.collection_name}/index'
data: json.encode(data)
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error creating index: ' + error_.status.error)
}
return json.decode(QDrantResponse[IndexOperationResponse], response.data)!
}
// Parameters for deleting an index
@[params]
pub struct DeleteIndexParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
field_name string @[json: 'field_name'; required] // Name of the field to delete index for
wait ?bool @[json: 'wait'] // Whether to wait until the changes have been applied
}
// Delete an index for a field in a collection
pub fn (mut self QDrantClient) delete_index(params DeleteIndexParams) !QDrantResponse[IndexOperationResponse] {
mut http_conn := self.httpclient()!
mut url := '/collections/${params.collection_name}/index/${params.field_name}'
if params.wait != none {
url += '?wait=${params.wait}'
}
req := httpconnection.Request{
method: .delete
prefix: url
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error deleting index: ' + error_.status.error)
}
return json.decode(QDrantResponse[IndexOperationResponse], response.data)!
}

View File

@@ -24,6 +24,53 @@ pub mut:
order_value f64 // Order value order_value f64 // Order value
} }
// Parameters for scrolling through points
@[params]
pub struct ScrollPointsParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
filter ?Filter @[json: 'filter'] // Filter conditions
limit int = 10 @[json: 'limit'] // Max number of results
offset ?string @[json: 'offset'] // Offset from which to continue scrolling
with_payload ?bool @[json: 'with_payload'] // Whether to include payload in the response
with_vector ?bool @[json: 'with_vector'] // Whether to include vectors in the response
}
// Response structure for scroll operation
pub struct ScrollResponse {
pub mut:
points []PointStruct @[json: 'points'] // List of points
next_page_offset ?string @[json: 'next_page_offset'] // Offset for the next page
}
// Point structure for scroll results
pub struct PointStruct {
pub mut:
id string @[json: 'id'] // Point ID
payload ?map[string]string @[json: 'payload'] // Payload key-value pairs (optional)
vector ?[]f64 @[json: 'vector'] // Vector data (optional)
}
// Scroll through points with pagination
pub fn (mut self QDrantClient) scroll_points(params ScrollPointsParams) !QDrantResponse[ScrollResponse] {
mut http_conn := self.httpclient()!
req := httpconnection.Request{
method: .post
prefix: '/collections/${params.collection_name}/points/scroll'
data: json.encode(params)
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error scrolling points: ' + error_.status.error)
}
return json.decode(QDrantResponse[ScrollResponse], response.data)!
}
// Retrieves all details from multiple points. // Retrieves all details from multiple points.
pub fn (mut self QDrantClient) retrieve_points(params RetrievePointsParams) !QDrantResponse[RetrievePointsResponse] { pub fn (mut self QDrantClient) retrieve_points(params RetrievePointsParams) !QDrantResponse[RetrievePointsResponse] {
mut http_conn := self.httpclient()! mut http_conn := self.httpclient()!
@@ -50,6 +97,7 @@ pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection collection_name string @[json: 'collection_name'; required] // Name of the collection
points []Point @[json: 'points'; required] // List of points to upsert points []Point @[json: 'points'; required] // List of points to upsert
shard_key ?string // Optional shard key for sharding shard_key ?string // Optional shard key for sharding
wait ?bool // Whether to wait until the changes have been applied
} }
// Represents a single point to be upserted. // Represents a single point to be upserted.
@@ -86,3 +134,309 @@ pub fn (mut self QDrantClient) upsert_points(params UpsertPointsParams) !QDrantR
return json.decode(QDrantResponse[UpsertPointsResponse], response.data)! return json.decode(QDrantResponse[UpsertPointsResponse], response.data)!
} }
// Parameters for getting a point by ID
@[params]
pub struct GetPointParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
id string @[json: 'id'; required] // ID of the point to retrieve
with_payload ?bool // Whether to include payload in the response
with_vector ?bool // Whether to include vector in the response
}
// Response structure for the get point operation
pub struct GetPointResponse {
pub mut:
id string // Point ID
payload map[string]string // Payload key-value pairs
vector ?[]f64 // Vector data (optional)
}
// Get a point by ID
pub fn (mut self QDrantClient) get_point(params GetPointParams) !QDrantResponse[GetPointResponse] {
mut http_conn := self.httpclient()!
mut url := '/collections/${params.collection_name}/points/${params.id}'
// Add query parameters if provided
mut query_params := []string{}
if params.with_payload != none {
query_params << 'with_payload=${params.with_payload}'
}
if params.with_vector != none {
query_params << 'with_vector=${params.with_vector}'
}
if query_params.len > 0 {
url += '?' + query_params.join('&')
}
req := httpconnection.Request{
method: .get
prefix: url
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error getting point: ' + error_.status.error)
}
return json.decode(QDrantResponse[GetPointResponse], response.data)!
}
// Filter condition for field matching
pub struct FieldCondition {
pub mut:
key string @[json: 'key'; required] // Field name to filter by
match_ ?string @[json: 'match'] // Exact match value (string)
match_integer ?int @[json: 'match'] // Exact match value (integer)
match_float ?f64 @[json: 'match'] // Exact match value (float)
match_bool ?bool @[json: 'match'] // Exact match value (boolean)
range ?Range @[json: 'range'] // Range condition
}
// Range condition for numeric fields
pub struct Range {
pub mut:
lt ?f64 @[json: 'lt'] // Less than
gt ?f64 @[json: 'gt'] // Greater than
gte ?f64 @[json: 'gte'] // Greater than or equal
lte ?f64 @[json: 'lte'] // Less than or equal
}
// Filter structure for search operations
pub struct Filter {
pub mut:
must ?[]FieldCondition @[json: 'must'] // All conditions must match
must_not ?[]FieldCondition @[json: 'must_not'] // None of the conditions should match
should ?[]FieldCondition @[json: 'should'] // At least one condition should match
}
// Parameters for searching points
@[params]
pub struct SearchParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
vector []f64 @[json: 'vector'; required] // Vector to search for
filter ?Filter @[json: 'filter'] // Filter conditions
limit int = 10 @[json: 'limit'] // Max number of results
offset ?int @[json: 'offset'] // Offset of the first result to return
with_payload ?bool @[json: 'with_payload'] // Whether to include payload in the response
with_vector ?bool @[json: 'with_vector'] // Whether to include vectors in the response
score_threshold ?f64 @[json: 'score_threshold'] // Minimal score threshold
}
// Scored point in search results
pub struct ScoredPoint {
pub mut:
id string @[json: 'id'] // Point ID
payload ?map[string]string @[json: 'payload'] // Payload key-value pairs (optional)
vector ?[]f64 @[json: 'vector'] // Vector data (optional)
score f64 @[json: 'score'] // Similarity score
}
// Response structure for search operation
pub struct SearchResponse {
pub mut:
points []ScoredPoint @[json: 'points'] // List of scored points
}
// Search for points based on vector similarity
pub fn (mut self QDrantClient) search(params SearchParams) !QDrantResponse[SearchResponse] {
mut http_conn := self.httpclient()!
req := httpconnection.Request{
method: .post
prefix: '/collections/${params.collection_name}/points/search'
data: json.encode(params)
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error searching points: ' + error_.status.error)
}
return json.decode(QDrantResponse[SearchResponse], response.data)!
}
// Points selector for delete operation
pub struct PointsSelector {
pub mut:
points ?[]string @[json: 'points'] // List of point IDs to delete
filter ?Filter @[json: 'filter'] // Filter condition to select points for deletion
}
// Parameters for deleting points
@[params]
pub struct DeletePointsParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
points_selector PointsSelector @[json: 'points_selector'; required] // Points selector
wait ?bool @[json: 'wait'] // Whether to wait until the changes have been applied
}
// Response structure for delete points operation
pub struct DeletePointsResponse {
pub mut:
status string @[json: 'status']
operation_id int @[json: 'operation_id']
}
// Delete points from a collection
pub fn (mut self QDrantClient) delete_points(params DeletePointsParams) !QDrantResponse[DeletePointsResponse] {
mut http_conn := self.httpclient()!
req := httpconnection.Request{
method: .post
prefix: '/collections/${params.collection_name}/points/delete'
data: json.encode(params)
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error deleting points: ' + error_.status.error)
}
return json.decode(QDrantResponse[DeletePointsResponse], response.data)!
}
// Parameters for counting points
@[params]
pub struct CountPointsParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
filter ?Filter @[json: 'filter'] // Filter conditions
exact ?bool @[json: 'exact'] // Whether to calculate exact count
}
// Response structure for count operation
pub struct CountResponse {
pub mut:
count int @[json: 'count'] // Number of points matching the filter
}
// Count points in a collection
pub fn (mut self QDrantClient) count_points(params CountPointsParams) !QDrantResponse[CountResponse] {
mut http_conn := self.httpclient()!
req := httpconnection.Request{
method: .post
prefix: '/collections/${params.collection_name}/points/count'
data: json.encode(params)
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error counting points: ' + error_.status.error)
}
return json.decode(QDrantResponse[CountResponse], response.data)!
}
// Parameters for setting payload
@[params]
pub struct SetPayloadParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
payload map[string]string @[json: 'payload'; required] // Payload to set
points ?[]string @[json: 'points'] // List of point IDs to set payload for
filter ?Filter @[json: 'filter'] // Filter condition to select points
wait ?bool @[json: 'wait'] // Whether to wait until the changes have been applied
}
// Response structure for payload operations
pub struct PayloadOperationResponse {
pub mut:
status string @[json: 'status']
operation_id int @[json: 'operation_id']
}
// Set payload for points
pub fn (mut self QDrantClient) set_payload(params SetPayloadParams) !QDrantResponse[PayloadOperationResponse] {
mut http_conn := self.httpclient()!
req := httpconnection.Request{
method: .post
prefix: '/collections/${params.collection_name}/points/payload'
data: json.encode(params)
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error setting payload: ' + error_.status.error)
}
return json.decode(QDrantResponse[PayloadOperationResponse], response.data)!
}
// Parameters for deleting payload
@[params]
pub struct DeletePayloadParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
keys []string @[json: 'keys'; required] // List of payload keys to delete
points ?[]string @[json: 'points'] // List of point IDs to delete payload from
filter ?Filter @[json: 'filter'] // Filter condition to select points
wait ?bool @[json: 'wait'] // Whether to wait until the changes have been applied
}
// Delete payload for points
pub fn (mut self QDrantClient) delete_payload(params DeletePayloadParams) !QDrantResponse[PayloadOperationResponse] {
mut http_conn := self.httpclient()!
req := httpconnection.Request{
method: .post
prefix: '/collections/${params.collection_name}/points/payload/delete'
data: json.encode(params)
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error deleting payload: ' + error_.status.error)
}
return json.decode(QDrantResponse[PayloadOperationResponse], response.data)!
}
// Parameters for clearing payload
@[params]
pub struct ClearPayloadParams {
pub mut:
collection_name string @[json: 'collection_name'; required] // Name of the collection
points ?[]string @[json: 'points'] // List of point IDs to clear payload for
filter ?Filter @[json: 'filter'] // Filter condition to select points
wait ?bool @[json: 'wait'] // Whether to wait until the changes have been applied
}
// Clear payload for points
pub fn (mut self QDrantClient) clear_payload(params ClearPayloadParams) !QDrantResponse[PayloadOperationResponse] {
mut http_conn := self.httpclient()!
req := httpconnection.Request{
method: .post
prefix: '/collections/${params.collection_name}/points/payload/clear'
data: json.encode(params)
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error clearing payload: ' + error_.status.error)
}
return json.decode(QDrantResponse[PayloadOperationResponse], response.data)!
}

View File

@@ -1,6 +1,7 @@
module qdrant module qdrant
import freeflowuniverse.herolib.core.httpconnection import freeflowuniverse.herolib.core.httpconnection
import json
// QDrant usage // QDrant usage
pub struct QDrantUsage { pub struct QDrantUsage {
@@ -31,6 +32,56 @@ pub mut:
error string // Error message error string // Error message
} }
// Service information
pub struct ServiceInfo {
pub mut:
version string // Version of the Qdrant server
commit ?string // Git commit hash
}
// Health check response
pub struct HealthCheckResponse {
pub mut:
title string // Title of the health check
status string // Status of the health check
version string // Version of the Qdrant server
}
// Get service information
pub fn (mut self QDrantClient) get_service_info() !QDrantResponse[ServiceInfo] {
mut http_conn := self.httpclient()!
req := httpconnection.Request{
method: .get
prefix: '/telemetry'
}
mut response := http_conn.send(req)!
if response.code >= 400 {
error_ := json.decode(QDrantErrorResponse, response.data)!
return error('Error getting service info: ' + error_.status.error)
}
return json.decode(QDrantResponse[ServiceInfo], response.data)!
}
// Check health of the Qdrant server
pub fn (mut self QDrantClient) health_check() !bool {
mut http_conn := self.httpclient()!
req := httpconnection.Request{
method: .get
prefix: '/healthz'
}
mut response := http_conn.send(req)!
if response.code >= 400 {
return false
}
return true
}
// httpclient creates a new HTTP connection to the Qdrant API // httpclient creates a new HTTP connection to the Qdrant API
fn (mut self QDrantClient) httpclient() !&httpconnection.HTTPConnection { fn (mut self QDrantClient) httpclient() !&httpconnection.HTTPConnection {
mut http_conn := httpconnection.new( mut http_conn := httpconnection.new(

View File

@@ -1 +1,326 @@
module qdrant module qdrant
import os
fn test_client_creation() {
// Create a client with default settings
mut client := QDrantClient{
name: 'test_client'
url: 'http://localhost:6333'
}
assert client.name == 'test_client'
assert client.url == 'http://localhost:6333'
assert client.secret == ''
}
fn test_client_with_auth() {
// Create a client with authentication
mut client := QDrantClient{
name: 'auth_client'
url: 'http://localhost:6333'
secret: 'test_api_key'
}
assert client.name == 'auth_client'
assert client.url == 'http://localhost:6333'
assert client.secret == 'test_api_key'
}
// The following tests require a running Qdrant server
// They are commented out to avoid test failures when no server is available
/*
fn test_collection_operations() {
if os.getenv('QDRANT_TEST_URL') == '' {
println('Skipping test_collection_operations: QDRANT_TEST_URL not set')
return
}
mut client := QDrantClient{
name: 'test_client'
url: os.getenv('QDRANT_TEST_URL')
}
// Create a test collection
create_result := client.create_collection(
collection_name: 'test_collection'
size: 128
distance: 'cosine'
) or {
assert false, 'Failed to create collection: ${err}'
return
}
assert create_result.status == 'ok'
// Check if collection exists
exists_result := client.is_collection_exists(
collection_name: 'test_collection'
) or {
assert false, 'Failed to check collection existence: ${err}'
return
}
assert exists_result.result.exists == true
// Get collection info
get_result := client.get_collection(
collection_name: 'test_collection'
) or {
assert false, 'Failed to get collection: ${err}'
return
}
assert get_result.result.config.params.vectors.size == 128
assert get_result.result.config.params.vectors.distance == 'cosine'
// Create an index
create_index_result := client.create_index(
collection_name: 'test_collection'
field_name: 'category'
field_schema: FieldSchema{
field_type: 'keyword'
}
wait: true
) or {
assert false, 'Failed to create index: ${err}'
return
}
assert create_index_result.status == 'ok'
// Delete the index
delete_index_result := client.delete_index(
collection_name: 'test_collection'
field_name: 'category'
wait: true
) or {
assert false, 'Failed to delete index: ${err}'
return
}
assert delete_index_result.status == 'ok'
// List collections
list_result := client.list_collections() or {
assert false, 'Failed to list collections: ${err}'
return
}
assert 'test_collection' in list_result.result.collections.map(it.collection_name)
// Delete collection
delete_result := client.delete_collection(
collection_name: 'test_collection'
) or {
assert false, 'Failed to delete collection: ${err}'
return
}
assert delete_result.status == 'ok'
}
fn test_points_operations() {
if os.getenv('QDRANT_TEST_URL') == '' {
println('Skipping test_points_operations: QDRANT_TEST_URL not set')
return
}
mut client := QDrantClient{
name: 'test_client'
url: os.getenv('QDRANT_TEST_URL')
}
// Create a test collection
client.create_collection(
collection_name: 'test_points'
size: 4
distance: 'cosine'
) or {
assert false, 'Failed to create collection: ${err}'
return
}
// Upsert points
points := [
Point{
id: '1'
vector: [f64(0.1), 0.2, 0.3, 0.4]
payload: {
'color': 'red'
'category': 'furniture'
}
},
Point{
id: '2'
vector: [f64(0.2), 0.3, 0.4, 0.5]
payload: {
'color': 'blue'
'category': 'electronics'
}
}
]
upsert_result := client.upsert_points(
collection_name: 'test_points'
points: points
wait: true
) or {
assert false, 'Failed to upsert points: ${err}'
return
}
assert upsert_result.status == 'ok'
// Get a point
get_result := client.get_point(
collection_name: 'test_points'
id: '1'
with_payload: true
with_vector: true
) or {
assert false, 'Failed to get point: ${err}'
return
}
assert get_result.result.id == '1'
assert get_result.result.payload['color'] == 'red'
// Search for points
search_result := client.search(
collection_name: 'test_points'
vector: [f64(0.1), 0.2, 0.3, 0.4]
limit: 10
) or {
assert false, 'Failed to search points: ${err}'
return
}
assert search_result.result.points.len > 0
// Scroll through points
scroll_result := client.scroll_points(
collection_name: 'test_points'
limit: 10
with_payload: true
with_vector: true
) or {
assert false, 'Failed to scroll points: ${err}'
return
}
assert scroll_result.result.points.len > 0
// Count points
count_result := client.count_points(
collection_name: 'test_points'
) or {
assert false, 'Failed to count points: ${err}'
return
}
assert count_result.result.count == 2
// Set payload
set_payload_result := client.set_payload(
collection_name: 'test_points'
payload: {
'price': '100'
'in_stock': 'true'
}
points: ['1']
) or {
assert false, 'Failed to set payload: ${err}'
return
}
assert set_payload_result.status == 'ok'
// Get point to verify payload was set
get_result_after_set := client.get_point(
collection_name: 'test_points'
id: '1'
with_payload: true
) or {
assert false, 'Failed to get point after setting payload: ${err}'
return
}
assert get_result_after_set.result.payload['price'] == '100'
assert get_result_after_set.result.payload['in_stock'] == 'true'
// Delete specific payload key
delete_payload_result := client.delete_payload(
collection_name: 'test_points'
keys: ['price']
points: ['1']
) or {
assert false, 'Failed to delete payload: ${err}'
return
}
assert delete_payload_result.status == 'ok'
// Clear all payload
clear_payload_result := client.clear_payload(
collection_name: 'test_points'
points: ['1']
) or {
assert false, 'Failed to clear payload: ${err}'
return
}
assert clear_payload_result.status == 'ok'
// Delete points
delete_result := client.delete_points(
collection_name: 'test_points'
points_selector: PointsSelector{
points: ['1', '2']
}
wait: true
) or {
assert false, 'Failed to delete points: ${err}'
return
}
assert delete_result.status == 'ok'
// Clean up
client.delete_collection(
collection_name: 'test_points'
) or {
assert false, 'Failed to delete collection: ${err}'
return
}
}
fn test_service_operations() {
if os.getenv('QDRANT_TEST_URL') == '' {
println('Skipping test_service_operations: QDRANT_TEST_URL not set')
return
}
mut client := QDrantClient{
name: 'test_client'
url: os.getenv('QDRANT_TEST_URL')
}
// Get service info
info_result := client.get_service_info() or {
assert false, 'Failed to get service info: ${err}'
return
}
assert info_result.result.version != ''
// Check health
health_result := client.health_check() or {
assert false, 'Failed to check health: ${err}'
return
}
assert health_result == true
}
*/