This commit is contained in:
2025-03-11 20:41:33 +01:00
parent 9448ae85cf
commit 237f9bd742
21 changed files with 15791 additions and 458 deletions

View File

@@ -195,13 +195,13 @@ fn initname() !string
e.g. systemd, bash, zinit
fn ipaddr_pub_get() !string
Returns the ipaddress as known on the public side is using resolver4.opendns.com
fn is_linux() bool
fn is_linux()! bool
fn is_linux_arm()! bool
fn is_linux_intel() bool
fn is_osx() bool
fn is_osx_arm() bool
fn is_osx_intel() bool
fn is_ubuntu() bool
fn is_linux_intel()! bool
fn is_osx()! bool
fn is_osx_arm()! bool
fn is_osx_intel()! bool
fn is_ubuntu()! bool
fn load_env_file(file_path string) !
fn memdb_exists(key string) bool
fn memdb_get(key string) string

View File

@@ -0,0 +1,9 @@
#!/usr/bin/env -S v -n -w -cg -gc none -cc tcc -d use_openssl -enable-globals run
import freeflowuniverse.herolib.installers.db.qdrant as qdrant_installer
mut db := qdrant_installer.get()!
db.install()!
db.start()!

View File

@@ -16,3 +16,26 @@ data = {
response = requests.post(url, headers=headers, data=json.dumps(data))
print(response.json())
#OTHER EXAMPLE WITH MORE ARGUMENTS
url = "https://s.jina.ai/"
params = {
"q": "Jina AI",
"gl": "US",
"hl": "en",
"num": 10,
"page": 1,
"location": "gent"
}
headers = {
"Accept": "application/json",
"Authorization": "Bearer jina_275aefb6495643408d4c499fce548080w5rYjijHfHVBi_vtAqNY6LBk-woz",
"X-Return-Format": "markdown",
"X-Timeout": "10"
}
response = requests.get(url, params=params, headers=headers)
print(response.json())

View File

@@ -0,0 +1,8 @@
!!hero_code.generate_client
name:'qdrant'
classname:'QDrantClient'
singleton:0
default:1
hasconfig:1
reset:0

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,112 @@
module qdrant
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui.console
__global (
qdrant_global map[string]&QDrantClient
qdrant_default string
)
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string
}
fn args_get(args_ ArgsGet) ArgsGet {
mut args := args_
if args.name == '' {
args.name = 'default'
}
return args
}
pub fn get(args_ ArgsGet) !&QDrantClient {
mut context := base.context()!
mut args := args_get(args_)
mut obj := QDrantClient{}
if args.name !in qdrant_global {
if !exists(args)! {
set(obj)!
} else {
heroscript := context.hero_config_get('qdrant', args.name)!
mut obj_ := heroscript_loads(heroscript)!
set_in_mem(obj_)!
}
}
return qdrant_global[args.name] or {
println(qdrant_global)
// bug if we get here because should be in globals
panic('could not get config for qdrant with name, is bug:${args.name}')
}
}
// register the config for the future
pub fn set(o QDrantClient) ! {
set_in_mem(o)!
mut context := base.context()!
heroscript := heroscript_dumps(o)!
context.hero_config_set('qdrant', o.name, heroscript)!
}
// does the config exists?
pub fn exists(args_ ArgsGet) !bool {
mut context := base.context()!
mut args := args_get(args_)
return context.hero_config_exists('qdrant', args.name)
}
pub fn delete(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
context.hero_config_delete('qdrant', args.name)!
if args.name in qdrant_global {
// del qdrant_global[args.name]
}
}
// only sets in mem, does not set as config
fn set_in_mem(o QDrantClient) ! {
mut o2 := obj_init(o)!
qdrant_global[o.name] = &o2
qdrant_default = o.name
}
@[params]
pub struct PlayArgs {
pub mut:
heroscript string // if filled in then plbook will be made out of it
plbook ?playbook.PlayBook
reset bool
}
pub fn play(args_ PlayArgs) ! {
mut args := args_
mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
mut install_actions := plbook.find(filter: 'qdrant.configure')!
if install_actions.len > 0 {
for install_action in install_actions {
heroscript := install_action.heroscript()
mut obj2 := heroscript_loads(heroscript)!
set(obj2)!
}
}
}
// switch instance to be used for qdrant
pub fn switch(name string) {
qdrant_default = name
}
// helpers
@[params]
pub struct DefaultConfigArgs {
instance string = 'default'
}

View File

@@ -0,0 +1,36 @@
module qdrant
import freeflowuniverse.herolib.data.paramsparser
import freeflowuniverse.herolib.data.encoderhero
import os
pub const version = '0.0.0'
const singleton = false
const default = true
// THIS THE THE SOURCE OF THE INFORMATION OF THIS FILE, HERE WE HAVE THE CONFIG OBJECT CONFIGURED AND MODELLED
@[heap]
pub struct QDrantClient {
pub mut:
name string = 'default'
secret string
url string = "http://localhost:6333/"
}
// your checking & initialization code if needed
fn obj_init(mycfg_ QDrantClient) !QDrantClient {
mut mycfg := mycfg_
return mycfg
}
/////////////NORMALLY NO NEED TO TOUCH
pub fn heroscript_dumps(obj QDrantClient) !string {
return encoderhero.encode[QDrantClient](obj)!
}
pub fn heroscript_loads(heroscript string) !QDrantClient {
mut obj := encoderhero.decode[QDrantClient](heroscript)!
return obj
}

View File

@@ -0,0 +1,30 @@
# qdrant
To get started
```vlang
import freeflowuniverse.herolib.clients. qdrant
mut client:= qdrant.get()!
client...
```
## example heroscript
```hero
!!qdrant.configure
secret: '...'
host: 'localhost'
port: 8888
```

View File

@@ -115,13 +115,13 @@ fn install() ! {
console.print_header('install ${args.name}')
//THIS IS EXAMPLE CODEAND NEEDS TO BE CHANGED
// mut url := ''
// if core.is_linux_arm() {
// if core.is_linux_arm()! {
// url = 'https://github.com/${args.name}-dev/${args.name}/releases/download/v??{version}/${args.name}_??{version}_linux_arm64.tar.gz'
// } else if core.is_linux_intel() {
// } else if core.is_linux_intel()! {
// url = 'https://github.com/${args.name}-dev/${args.name}/releases/download/v??{version}/${args.name}_??{version}_linux_amd64.tar.gz'
// } else if core.is_osx_arm() {
// } else if core.is_osx_arm()! {
// url = 'https://github.com/${args.name}-dev/${args.name}/releases/download/v??{version}/${args.name}_??{version}_darwin_arm64.tar.gz'
// } else if osal.is_osx_intel() {
// } else if osal.is_osx_intel()! {
// url = 'https://github.com/${args.name}-dev/${args.name}/releases/download/v??{version}/${args.name}_??{version}_darwin_amd64.tar.gz'
// } else {
// return error('unsported platform')

View File

@@ -7,7 +7,7 @@ import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui.console
@if args.cat == .installer
import freeflowuniverse.herolib.sysadmin.startupmanager
import freeflowuniverse.herolib.osal.startupmanager
import freeflowuniverse.herolib.osal.zinit
@if args.startupmanager
import time

View File

@@ -0,0 +1,13 @@
!!hero_code.generate_installer
name:'qdrant'
classname:'QDrant'
singleton:0
templates:1
default:1
title:''
supported_platforms:''
reset:0
startupmanager:1
hasconfig:1
build:1

View File

@@ -0,0 +1,149 @@
module qdrant
import freeflowuniverse.herolib.osal
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.core
import freeflowuniverse.herolib.core.texttools
// import freeflowuniverse.herolib.core.pathlib
// import freeflowuniverse.herolib.osal.systemd
import freeflowuniverse.herolib.osal.zinit
import freeflowuniverse.herolib.installers.ulist
// import freeflowuniverse.herolib.installers.lang.golang
// import freeflowuniverse.herolib.installers.lang.rust
// import freeflowuniverse.herolib.installers.lang.python
import freeflowuniverse.herolib.core.httpconnection
import os
fn startupcmd() ![]zinit.ZProcessNewArgs {
mut res := []zinit.ZProcessNewArgs{}
res << zinit.ZProcessNewArgs{
name: 'qdrant'
cmd: 'qdrant --config-path ${os.home_dir()}/hero/var/qdrant/config.yaml'
}
return res
}
fn running() !bool {
println("running")
mut installer := get()!
url:='curl http://localhost:6333'
mut conn := httpconnection.new(name: 'qdrant', url: url)!
r := conn.get(prefix: 'healthz', debug: false) or {return false}
println(r)
return false
}
fn start_pre() ! {
}
fn start_post() ! {
}
fn stop_pre() ! {
}
fn stop_post() ! {
}
//////////////////// following actions are not specific to instance of the object
// checks if a certain version or above is installed
fn installed() !bool {
res := os.execute('${osal.profile_path_source_and()!} qdrant -V')
if res.exit_code != 0 {
println("Error to call qdrant: ${res}")
return false
}
r := res.output.split_into_lines().filter(it.contains("qdrant"))
if r.len != 1 {
return error("couldn't parse qdrant version.\n${res.output}")
}
if texttools.version(version) == texttools.version(r[0].all_after("qdrant")) {
return true
}
return false
}
// get the Upload List of the files
fn ulist_get() !ulist.UList {
return ulist.UList{}
}
// uploads to S3 server if configured
fn upload() ! {
// installers.upload(
// cmdname: 'qdrant'
// source: '${gitpath}/target/x86_64-unknown-linux-musl/release/qdrant'
// )!
}
fn install() ! {
console.print_header('install qdrant')
mut url := ''
if core.is_linux_arm()! {
url = 'https://github.com/qdrant/qdrant/releases/download/v${version}/qdrant-aarch64-unknown-linux-musl.tar.gz'
} else if core.is_linux_intel()! {
url = 'https://github.com/qdrant/qdrant/releases/download/v${version}/qdrant-x86_64-unknown-linux-musl.tar.gz'
} else if core.is_osx_arm()! {
url = 'https://github.com/qdrant/qdrant/releases/download/v${version}/qdrant-aarch64-apple-darwin.tar.gz'
} else if core.is_osx_intel()! {
url = 'https://github.com/qdrant/qdrant/releases/download/v${version}/qdrant-x86_64-apple-darwin.tar.gz'
} else {
return error('unsported platform')
}
mut dest := osal.download(
url: url
minsize_kb: 18000
expand_dir: '/tmp/qdrant'
)!
mut binpath := dest.file_get('qdrant')!
osal.cmd_add(
cmdname: 'qdrant'
source: binpath.path
)!
}
fn build() ! {
// url := 'https://github.com/threefoldtech/qdrant'
// make sure we install base on the node
// if osal.platform() != .ubuntu {
// return error('only support ubuntu for now')
// }
// golang.install()!
// console.print_header('build qdrant')
// gitpath := gittools.get_repo(coderoot: '/tmp/builder', url: url, reset: true, pull: true)!
// cmd := '
// cd ${gitpath}
// source ~/.cargo/env
// exit 1 #todo
// '
// osal.execute_stdout(cmd)!
//
// //now copy to the default bin path
// mut binpath := dest.file_get('...')!
// adds it to path
// osal.cmd_add(
// cmdname: 'griddriver2'
// source: binpath.path
// )!
}
fn destroy() ! {
osal.process_kill_recursive(name:'qdrant')!
osal.cmd_delete('qdrant')!
osal.package_remove('
qdrant
')!
osal.rm("
qdrant
${os.home_dir()}/hero/var/qdrant
")!
}

View File

@@ -0,0 +1,279 @@
module qdrant
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.osal.startupmanager
import freeflowuniverse.herolib.osal.zinit
import time
__global (
qdrant_global map[string]&QDrant
qdrant_default string
)
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string
}
fn args_get(args_ ArgsGet) ArgsGet {
mut args := args_
if args.name == '' {
args.name = 'default'
}
return args
}
pub fn get(args_ ArgsGet) !&QDrant {
mut context := base.context()!
mut args := args_get(args_)
mut obj := QDrant{}
if args.name !in qdrant_global {
if !exists(args)! {
set(obj)!
} else {
heroscript := context.hero_config_get('qdrant', args.name)!
mut obj_ := heroscript_loads(heroscript)!
set_in_mem(obj_)!
}
}
return qdrant_global[args.name] or {
println(qdrant_global)
// bug if we get here because should be in globals
panic('could not get config for qdrant with name, is bug:${args.name}')
}
}
// register the config for the future
pub fn set(o QDrant) ! {
set_in_mem(o)!
mut context := base.context()!
heroscript := heroscript_dumps(o)!
context.hero_config_set('qdrant', o.name, heroscript)!
}
// does the config exists?
pub fn exists(args_ ArgsGet) !bool {
mut context := base.context()!
mut args := args_get(args_)
return context.hero_config_exists('qdrant', args.name)
}
pub fn delete(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
context.hero_config_delete('qdrant', args.name)!
if args.name in qdrant_global {
// del qdrant_global[args.name]
}
}
// only sets in mem, does not set as config
fn set_in_mem(o QDrant) ! {
mut o2 := obj_init(o)!
qdrant_global[o.name] = &o2
qdrant_default = o.name
}
@[params]
pub struct PlayArgs {
pub mut:
heroscript string // if filled in then plbook will be made out of it
plbook ?playbook.PlayBook
reset bool
}
pub fn play(args_ PlayArgs) ! {
mut args := args_
mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
mut install_actions := plbook.find(filter: 'qdrant.configure')!
if install_actions.len > 0 {
for install_action in install_actions {
heroscript := install_action.heroscript()
mut obj2 := heroscript_loads(heroscript)!
set(obj2)!
}
}
mut other_actions := plbook.find(filter: 'qdrant.')!
for other_action in other_actions {
if other_action.name in ['destroy', 'install', 'build'] {
mut p := other_action.params
reset := p.get_default_false('reset')
if other_action.name == 'destroy' || reset {
console.print_debug('install action qdrant.destroy')
destroy()!
}
if other_action.name == 'install' {
console.print_debug('install action qdrant.install')
install()!
}
}
if other_action.name in ['start', 'stop', 'restart'] {
mut p := other_action.params
name := p.get('name')!
mut qdrant_obj := get(name: name)!
console.print_debug('action object:\n${qdrant_obj}')
if other_action.name == 'start' {
console.print_debug('install action qdrant.${other_action.name}')
qdrant_obj.start()!
}
if other_action.name == 'stop' {
console.print_debug('install action qdrant.${other_action.name}')
qdrant_obj.stop()!
}
if other_action.name == 'restart' {
console.print_debug('install action qdrant.${other_action.name}')
qdrant_obj.restart()!
}
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////# LIVE CYCLE MANAGEMENT FOR INSTALLERS ///////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////
fn startupmanager_get(cat zinit.StartupManagerType) !startupmanager.StartupManager {
// unknown
// screen
// zinit
// tmux
// systemd
match cat {
.zinit {
console.print_debug('startupmanager: zinit')
return startupmanager.get(cat: .zinit)!
}
.systemd {
console.print_debug('startupmanager: systemd')
return startupmanager.get(cat: .systemd)!
}
else {
console.print_debug('startupmanager: auto')
return startupmanager.get()!
}
}
}
// load from disk and make sure is properly intialized
pub fn (mut self QDrant) reload() ! {
switch(self.name)
self = obj_init(self)!
}
pub fn (mut self QDrant) start() ! {
switch(self.name)
if self.running()! {
return
}
console.print_header('qdrant start')
if !installed()! {
install()!
}
configure()!
start_pre()!
for zprocess in startupcmd()! {
mut sm := startupmanager_get(zprocess.startuptype)!
console.print_debug('starting qdrant with ${zprocess.startuptype}...')
sm.new(zprocess)!
sm.start(zprocess.name)!
}
start_post()!
for _ in 0 .. 50 {
if self.running()! {
return
}
time.sleep(100 * time.millisecond)
}
return error('qdrant did not install properly.')
}
pub fn (mut self QDrant) install_start(args InstallArgs) ! {
switch(self.name)
self.install(args)!
self.start()!
}
pub fn (mut self QDrant) stop() ! {
switch(self.name)
stop_pre()!
for zprocess in startupcmd()! {
mut sm := startupmanager_get(zprocess.startuptype)!
sm.stop(zprocess.name)!
}
stop_post()!
}
pub fn (mut self QDrant) restart() ! {
switch(self.name)
self.stop()!
self.start()!
}
pub fn (mut self QDrant) running() !bool {
switch(self.name)
// walk over the generic processes, if not running return
for zprocess in startupcmd()! {
mut sm := startupmanager_get(zprocess.startuptype)!
r := sm.running(zprocess.name)!
if r == false {
return false
}
}
return running()!
}
@[params]
pub struct InstallArgs {
pub mut:
reset bool
}
pub fn (mut self QDrant) install(args InstallArgs) ! {
switch(self.name)
if args.reset || (!installed()!) {
install()!
}
}
pub fn (mut self QDrant) build() ! {
switch(self.name)
build()!
}
pub fn (mut self QDrant) destroy() ! {
switch(self.name)
self.stop() or {}
destroy()!
}
// switch instance to be used for qdrant
pub fn switch(name string) {
qdrant_default = name
}
// helpers
@[params]
pub struct DefaultConfigArgs {
instance string = 'default'
}

View File

@@ -0,0 +1,50 @@
module qdrant
import os
import freeflowuniverse.herolib.data.encoderhero
import freeflowuniverse.herolib.core.pathlib
import freeflowuniverse.herolib.ui.console
pub const version = '1.13.4'
const singleton = false
const default = true
// THIS THE THE SOURCE OF THE INFORMATION OF THIS FILE, HERE WE HAVE THE CONFIG OBJECT CONFIGURED AND MODELLED
@[heap]
pub struct QDrant {
pub mut:
name string = 'default'
homedir string
configpath string
username string
password string @[secret]
title string
host string
port int
}
// your checking & initialization code if needed
fn obj_init(mycfg_ QDrant) !QDrant {
mut mycfg := mycfg_
return mycfg
}
// called before start if done
fn configure() ! {
mut installer := get()!
storage_path:="${os.home_dir()}/hero/var/qdrant"
mut mycode := $tmpl('templates/config.yaml')
mut path := pathlib.get_file(path: "${os.home_dir()}/hero/var/qdrant/config.yaml", create: true)!
path.write(mycode)!
// console.print_debug(mycode)
}
/////////////NORMALLY NO NEED TO TOUCH
pub fn heroscript_dumps(obj QDrant) !string {
return encoderhero.encode[QDrant](obj)!
}
pub fn heroscript_loads(heroscript string) !QDrant {
mut obj := encoderhero.decode[QDrant](heroscript)!
return obj
}

View File

@@ -0,0 +1,44 @@
# qdrant
To get started
```vlang
import freeflowuniverse.herolib.installers.something.qdrant as qdrant_installer
heroscript:="
!!qdrant.configure name:'test'
password: '1234'
port: 7701
!!qdrant.start name:'test' reset:1
"
qdrant_installer.play(heroscript=heroscript)!
//or we can call the default and do a start with reset
//mut installer:= qdrant_installer.get()!
//installer.start(reset:true)!
```
## example heroscript
```hero
!!qdrant.configure
homedir: '/home/user/qdrant'
username: 'admin'
password: 'secretpassword'
title: 'Some Title'
host: 'localhost'
port: 8888
```

View File

@@ -0,0 +1,353 @@
log_level: INFO
# Logging configuration
# Qdrant logs to stdout. You may configure to also write logs to a file on disk.
# Be aware that this file may grow indefinitely.
# logger:
# # Logging format, supports `text` and `json`
# format: text
# on_disk:
# enabled: true
# log_file: path/to/log/file.log
# log_level: INFO
# # Logging format, supports `text` and `json`
# format: text
storage:
# Where to store all the data
storage_path: ${storage_path}/storage
# Where to store snapshots
snapshots_path: ${storage_path}/snapshots
snapshots_config:
# "local" or "s3" - where to store snapshots
snapshots_storage: local
# s3_config:
# bucket: ""
# region: ""
# access_key: ""
# secret_key: ""
# Where to store temporary files
# If null, temporary snapshots are stored in: storage/snapshots_temp/
temp_path: null
# If true - point payloads will not be stored in memory.
# It will be read from the disk every time it is requested.
# This setting saves RAM by (slightly) increasing the response time.
# Note: those payload values that are involved in filtering and are indexed - remain in RAM.
#
# Default: true
on_disk_payload: true
# Maximum number of concurrent updates to shard replicas
# If `null` - maximum concurrency is used.
update_concurrency: null
# Write-ahead-log related configuration
wal:
# Size of a single WAL segment
wal_capacity_mb: 32
# Number of WAL segments to create ahead of actual data requirement
wal_segments_ahead: 0
# Normal node - receives all updates and answers all queries
node_type: "Normal"
# Listener node - receives all updates, but does not answer search/read queries
# Useful for setting up a dedicated backup node
# node_type: "Listener"
performance:
# Number of parallel threads used for search operations. If 0 - auto selection.
max_search_threads: 1
# Max number of threads (jobs) for running optimizations across all collections, each thread runs one job.
# If 0 - have no limit and choose dynamically to saturate CPU.
# Note: each optimization job will also use `max_indexing_threads` threads by itself for index building.
max_optimization_threads: 1
# CPU budget, how many CPUs (threads) to allocate for an optimization job.
# If 0 - auto selection, keep 1 or more CPUs unallocated depending on CPU size
# If negative - subtract this number of CPUs from the available CPUs.
# If positive - use this exact number of CPUs.
optimizer_cpu_budget: 1
# Prevent DDoS of too many concurrent updates in distributed mode.
# One external update usually triggers multiple internal updates, which breaks internal
# timings. For example, the health check timing and consensus timing.
# If null - auto selection.
update_rate_limit: null
# Limit for number of incoming automatic shard transfers per collection on this node, does not affect user-requested transfers.
# The same value should be used on all nodes in a cluster.
# Default is to allow 1 transfer.
# If null - allow unlimited transfers.
#incoming_shard_transfers_limit: 1
# Limit for number of outgoing automatic shard transfers per collection on this node, does not affect user-requested transfers.
# The same value should be used on all nodes in a cluster.
# Default is to allow 1 transfer.
# If null - allow unlimited transfers.
#outgoing_shard_transfers_limit: 1
# Enable async scorer which uses io_uring when rescoring.
# Only supported on Linux, must be enabled in your kernel.
# See: <https://qdrant.tech/articles/io_uring/#and-what-about-qdrant>
#async_scorer: false
optimizers:
# The minimal fraction of deleted vectors in a segment, required to perform segment optimization
deleted_threshold: 0.2
# The minimal number of vectors in a segment, required to perform segment optimization
vacuum_min_vector_number: 1000
# Target amount of segments optimizer will try to keep.
# Real amount of segments may vary depending on multiple parameters:
# - Amount of stored points
# - Current write RPS
#
# It is recommended to select default number of segments as a factor of the number of search threads,
# so that each segment would be handled evenly by one of the threads.
# If `default_segment_number = 0`, will be automatically selected by the number of available CPUs
default_segment_number: 0
# Do not create segments larger this size (in KiloBytes).
# Large segments might require disproportionately long indexation times,
# therefore it makes sense to limit the size of segments.
#
# If indexation speed have more priority for your - make this parameter lower.
# If search speed is more important - make this parameter higher.
# Note: 1Kb = 1 vector of size 256
# If not set, will be automatically selected considering the number of available CPUs.
max_segment_size_kb: null
# Maximum size (in KiloBytes) of vectors to store in-memory per segment.
# Segments larger than this threshold will be stored as read-only memmapped file.
# To enable memmap storage, lower the threshold
# Note: 1Kb = 1 vector of size 256
# To explicitly disable mmap optimization, set to `0`.
# If not set, will be disabled by default.
memmap_threshold_kb: null
# Maximum size (in KiloBytes) of vectors allowed for plain index.
# Default value based on https://github.com/google-research/google-research/blob/master/scann/docs/algorithms.md
# Note: 1Kb = 1 vector of size 256
# To explicitly disable vector indexing, set to `0`.
# If not set, the default value will be used.
indexing_threshold_kb: 20000
# Interval between forced flushes.
flush_interval_sec: 5
# Max number of threads (jobs) for running optimizations per shard.
# Note: each optimization job will also use `max_indexing_threads` threads by itself for index building.
# If null - have no limit and choose dynamically to saturate CPU.
# If 0 - no optimization threads, optimizations will be disabled.
max_optimization_threads: null
# This section has the same options as 'optimizers' above. All values specified here will overwrite the collections
# optimizers configs regardless of the config above and the options specified at collection creation.
#optimizers_overwrite:
# deleted_threshold: 0.2
# vacuum_min_vector_number: 1000
# default_segment_number: 0
# max_segment_size_kb: null
# memmap_threshold_kb: null
# indexing_threshold_kb: 20000
# flush_interval_sec: 5
# max_optimization_threads: null
# Default parameters of HNSW Index. Could be overridden for each collection or named vector individually
hnsw_index:
# Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.
m: 16
# Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build index.
ef_construct: 100
# Minimal size (in KiloBytes) of vectors for additional payload-based indexing.
# If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used -
# in this case full-scan search should be preferred by query planner and additional indexing is not required.
# Note: 1Kb = 1 vector of size 256
full_scan_threshold_kb: 10000
# Number of parallel threads used for background index building.
# If 0 - automatically select.
# Best to keep between 8 and 16 to prevent likelihood of building broken/inefficient HNSW graphs.
# On small CPUs, less threads are used.
max_indexing_threads: 0
# Store HNSW index on disk. If set to false, index will be stored in RAM. Default: false
on_disk: false
# Custom M param for hnsw graph built for payload index. If not set, default M will be used.
payload_m: null
# Default shard transfer method to use if none is defined.
# If null - don't have a shard transfer preference, choose automatically.
# If stream_records, snapshot or wal_delta - prefer this specific method.
# More info: https://qdrant.tech/documentation/guides/distributed_deployment/#shard-transfer-method
shard_transfer_method: null
# Default parameters for collections
collection:
# Number of replicas of each shard that network tries to maintain
replication_factor: 1
# How many replicas should apply the operation for us to consider it successful
write_consistency_factor: 1
# Default parameters for vectors.
vectors:
# Whether vectors should be stored in memory or on disk.
on_disk: null
# shard_number_per_node: 1
# Default quantization configuration.
# More info: https://qdrant.tech/documentation/guides/quantization
quantization: null
# Default strict mode parameters for newly created collections.
strict_mode:
# Whether strict mode is enabled for a collection or not.
enabled: false
# Max allowed `limit` parameter for all APIs that don't have their own max limit.
max_query_limit: null
# Max allowed `timeout` parameter.
max_timeout: null
# Allow usage of unindexed fields in retrieval based (eg. search) filters.
unindexed_filtering_retrieve: null
# Allow usage of unindexed fields in filtered updates (eg. delete by payload).
unindexed_filtering_update: null
# Max HNSW value allowed in search parameters.
search_max_hnsw_ef: null
# Whether exact search is allowed or not.
search_allow_exact: null
# Max oversampling value allowed in search.
search_max_oversampling: null
service:
# Maximum size of POST data in a single request in megabytes
max_request_size_mb: 32
# Number of parallel workers used for serving the api. If 0 - equal to the number of available cores.
# If missing - Same as storage.max_search_threads
max_workers: 0
# Host to bind the service on
host: 0.0.0.0
# HTTP(S) port to bind the service on
http_port: 6333
# gRPC port to bind the service on.
# If `null` - gRPC is disabled. Default: null
# Comment to disable gRPC:
grpc_port: 6334
# Enable CORS headers in REST API.
# If enabled, browsers would be allowed to query REST endpoints regardless of query origin.
# More info: https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS
# Default: true
enable_cors: true
# Enable HTTPS for the REST and gRPC API
enable_tls: false
# Check user HTTPS client certificate against CA file specified in tls config
verify_https_client_certificate: false
# Set an api-key.
# If set, all requests must include a header with the api-key.
# example header: `api-key: <API-KEY>`
#
# If you enable this you should also enable TLS.
# (Either above or via an external service like nginx.)
# Sending an api-key over an unencrypted channel is insecure.
#
# Uncomment to enable.
# api_key: your_secret_api_key_here
# Set an api-key for read-only operations.
# If set, all requests must include a header with the api-key.
# example header: `api-key: <API-KEY>`
#
# If you enable this you should also enable TLS.
# (Either above or via an external service like nginx.)
# Sending an api-key over an unencrypted channel is insecure.
#
# Uncomment to enable.
# read_only_api_key: your_secret_read_only_api_key_here
# Uncomment to enable JWT Role Based Access Control (RBAC).
# If enabled, you can generate JWT tokens with fine-grained rules for access control.
# Use generated token instead of API key.
#
# jwt_rbac: true
# Hardware reporting adds information to the API responses with a
# hint on how many resources were used to execute the request.
#
# Uncomment to enable.
# hardware_reporting: true
cluster:
# Use `enabled: true` to run Qdrant in distributed deployment mode
enabled: false
# Configuration of the inter-cluster communication
p2p:
# Port for internal communication between peers
port: 6335
# Use TLS for communication between peers
enable_tls: false
# Configuration related to distributed consensus algorithm
consensus:
# How frequently peers should ping each other.
# Setting this parameter to lower value will allow consensus
# to detect disconnected nodes earlier, but too frequent
# tick period may create significant network and CPU overhead.
# We encourage you NOT to change this parameter unless you know what you are doing.
tick_period_ms: 100
# Set to true to prevent service from sending usage statistics to the developers.
# Read more: https://qdrant.tech/documentation/guides/telemetry
telemetry_disabled: false
# TLS configuration.
# Required if either service.enable_tls or cluster.p2p.enable_tls is true.
# tls:
# # Server certificate chain file
# cert: ./tls/cert.pem
# # Server private key file
# key: ./tls/key.pem
# # Certificate authority certificate file.
# # This certificate will be used to validate the certificates
# # presented by other nodes during inter-cluster communication.
# #
# # If verify_https_client_certificate is true, it will verify
# # HTTPS client certificate
# #
# # Required if cluster.p2p.enable_tls is true.
# ca_cert: ./tls/cacert.pem
# # TTL in seconds to reload certificate from disk, useful for certificate rotations.
# # Only works for HTTPS endpoints. Does not support gRPC (and intra-cluster communication).
# # If `null` - TTL is disabled.
# cert_ttl: 3600

View File

@@ -98,6 +98,7 @@ pub fn (mut self ScreensFactory) add(args_ ScreenAddArgs) !Screen {
name: args.name
cmd: args.cmd
}
// println(self.screens)
if args.start {
self.start(args.name)!
}

View File

@@ -45,8 +45,8 @@ pub fn get(args StartupManagerArgs) !StartupManager {
if args.cat == .unknown {
if zinit.check() {
sm.cat = .zinit
} else if systemd.check()! {
sm.cat = .systemd
}else {
sm.cat = .screen
}
}
return sm
@@ -76,11 +76,11 @@ pub fn (mut sm StartupManager) new(args zinit.ZProcessNewArgs) ! {
match mycat {
.screen {
mut scr := screen.new(reset: false)!
console.print_debug('screen')
console.print_debug('screen startup manager ${args.name} cmd:${args.cmd}')
_ = scr.add(name: args.name, cmd: args.cmd, reset: args.restart)!
}
.systemd {
console.print_debug('systemd start ${args.name}')
// console.print_debug('systemd start ${args.name}')
mut systemdfactory := systemd.new()!
systemdfactory.new(
cmd: args.cmd
@@ -309,6 +309,14 @@ pub fn (mut sm StartupManager) output(name string) !string {
}
pub fn (mut sm StartupManager) exists(name string) !bool {
println(sm.cat)
if sm.cat == .unknown {
if zinit.check() {
sm.cat = .zinit
}else {
sm.cat = .screen
}
}
match sm.cat {
.screen {
mut scr := screen.new(reset: false) or { panic("can't get screen") }

View File

@@ -1,29 +0,0 @@
# startup manager
```go
import freeflowuniverse.herolib.sysadmin.startupmanager
mut sm:=startupmanager.get()!
sm.start(
name: 'myscreen'
cmd: 'htop'
description: '...'
)!
```
## some basic commands for screen
```bash
#list the screens
screen -ls
#attach to the screens
screen -r myscreen
```
to exit a screen to
```
ctrl a d
```

View File

@@ -1,380 +0,0 @@
module startupmanager
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.osal.screen
import freeflowuniverse.herolib.osal.systemd
import freeflowuniverse.herolib.osal.zinit
// // TODO: check if using this interface would simplify things
// pub interface StartupManagerI {
// new(args zinit.ZProcessNewArgs)!
// start(name string)!
// stop(name string)!
// restart(name string)!
// delete(name string)!
// status(name string) !ProcessStatus
// running(name string) !bool
// output(name string) !string
// exists(name string) !bool
// list_services() ![]string
// }
pub enum StartupManagerType {
unknown
screen
zinit
tmux
systemd
}
pub struct StartupManager {
pub mut:
cat StartupManagerType
}
@[params]
pub struct StartupManagerArgs {
pub mut:
cat StartupManagerType
}
pub fn get(args StartupManagerArgs) !StartupManager {
mut sm := StartupManager{
cat: args.cat
}
if args.cat == .unknown {
if zinit.check() {
sm.cat = .zinit
} else if systemd.check()! {
sm.cat = .systemd
}
}
return sm
}
// launch a new process
//```
// name string @[required]
// cmd string @[required]
// cmd_stop string
// cmd_test string //command line to test service is running
// status ZProcessStatus
// pid int
// after []string //list of service we depend on
// env map[string]string
// oneshot bool
// start bool = true
// restart bool = true // whether the process should be restarted on failure
// description string //not used in zinit
//```
pub fn (mut sm StartupManager) new(args zinit.ZProcessNewArgs) ! {
console.print_debug("startupmanager start:${args.name} cmd:'${args.cmd}' restart:${args.restart}")
mut mycat := sm.cat
if args.startuptype == .systemd {
mycat = .systemd
}
match mycat {
.screen {
mut scr := screen.new(reset: false)!
console.print_debug('screen')
_ = scr.add(name: args.name, cmd: args.cmd, reset: args.restart)!
}
.systemd {
console.print_debug('systemd start ${args.name}')
mut systemdfactory := systemd.new()!
systemdfactory.new(
cmd: args.cmd
name: args.name
description: args.description
start: args.start
restart: args.restart
env: args.env
)!
}
.zinit {
console.print_debug('zinit start ${args.name}')
mut zinitfactory := zinit.new()!
// pub struct ZProcessNewArgs {
// name string @[required]
// cmd string @[required]
// cmd_stop string
// cmd_test string
// cmd_file bool // if we wanna force to run it as a file which is given to bash -c (not just a cmd in zinit)
// test string
// test_file bool
// after []string
// env map[string]string
// oneshot bool
// }
zinitfactory.new(args)!
}
else {
panic('to implement, startup manager only support screen & systemd for now: ${mycat}')
}
}
// if args.start {
// sm.start(args.name)!
// } else if args.restart {
// sm.restart(args.name)!
// }
}
pub fn (mut sm StartupManager) start(name string) ! {
match sm.cat {
.screen {
return
}
.systemd {
console.print_debug('systemd process start ${name}')
mut systemdfactory := systemd.new()!
if systemdfactory.exists(name) {
// console.print_header("*************")
mut systemdprocess := systemdfactory.get(name)!
systemdprocess.start()!
} else {
return error('process in systemd with name ${name} not found')
}
}
.zinit {
console.print_debug('zinit process start ${name}')
mut zinitfactory := zinit.new()!
zinitfactory.start(name)!
}
else {
panic('to implement, startup manager only support screen for now: ${sm.cat}')
}
}
}
pub fn (mut sm StartupManager) stop(name string) ! {
match sm.cat {
.screen {
mut screen_factory := screen.new(reset: false)!
mut scr := screen_factory.get(name) or { return }
scr.cmd_send('^C')!
screen_factory.kill(name)!
}
.systemd {
console.print_debug('systemd stop ${name}')
mut systemdfactory := systemd.new()!
if systemdfactory.exists(name) {
mut systemdprocess := systemdfactory.get(name)!
systemdprocess.stop()!
}
}
.zinit {
console.print_debug('zinit stop ${name}')
mut zinitfactory := zinit.new()!
zinitfactory.load()!
if zinitfactory.exists(name) {
zinitfactory.stop(name)!
}
}
else {
panic('to implement, startup manager only support screen for now: ${sm.cat}')
}
}
}
// kill the process by name
pub fn (mut sm StartupManager) restart(name string) ! {
match sm.cat {
.screen {
panic('implement')
}
.systemd {
console.print_debug('systemd restart ${name}')
mut systemdfactory := systemd.new()!
mut systemdprocess := systemdfactory.get(name)!
systemdprocess.restart()!
}
.zinit {
console.print_debug('zinit restart ${name}')
mut zinitfactory := zinit.new()!
zinitfactory.stop(name)!
zinitfactory.start(name)!
}
else {
panic('to implement, startup manager only support screen for now: ${sm.cat}')
}
}
}
// remove from the startup manager
pub fn (mut sm StartupManager) delete(name string) ! {
match sm.cat {
.screen {
mut screen_factory := screen.new(reset: false)!
mut scr := screen_factory.get(name) or { return }
scr.cmd_send('^C')!
screen_factory.kill(name)!
}
.systemd {
mut systemdfactory := systemd.new()!
mut systemdprocess := systemdfactory.get(name)!
systemdprocess.delete()!
}
.zinit {
mut zinitfactory := zinit.new()!
zinitfactory.load()!
if zinitfactory.exists(name) {
zinitfactory.delete(name)!
}
}
else {
panic('to implement, startup manager only support screen & systemd for now: ${sm.cat}')
}
}
}
pub enum ProcessStatus {
unknown
active
inactive
failed
activating
deactivating
}
// remove from the startup manager
pub fn (mut sm StartupManager) status(name string) !ProcessStatus {
match sm.cat {
.screen {
mut screen_factory := screen.new(reset: false)!
mut scr := screen_factory.get(name) or {
return error('process with name ${name} not found')
}
match scr.status()! {
.active { return .active }
.inactive { return .inactive }
.unknown { return .unknown }
}
}
.systemd {
mut systemdfactory := systemd.new()!
mut systemdprocess := systemdfactory.get(name) or { return .unknown }
systemd_status := systemdprocess.status() or {
return error('Failed to get status of process ${name}\n${err}')
}
s := ProcessStatus.from(systemd_status.str())!
return s
}
.zinit {
mut zinitfactory := zinit.new()!
mut p := zinitfactory.get(name) or { return .unknown }
// unknown
// init
// ok
// killed
// error
// blocked
// spawned
match mut p.status()! {
.init { return .activating }
.ok { return .active }
.error { return .failed }
.blocked { return .inactive }
.killed { return .inactive }
.spawned { return .activating }
.unknown { return .unknown }
}
}
else {
panic('to implement, startup manager only support screen & systemd for now: ${sm.cat}')
}
}
}
pub fn (mut sm StartupManager) running(name string) !bool {
if !sm.exists(name)! {
return false
}
mut s := sm.status(name)!
return s == .active
}
// remove from the startup manager
pub fn (mut sm StartupManager) output(name string) !string {
match sm.cat {
.screen {
panic('implement')
}
.systemd {
return systemd.journalctl(service: name)!
}
else {
panic('to implement, startup manager only support screen & systemd for now')
}
}
}
pub fn (mut sm StartupManager) exists(name string) !bool {
match sm.cat {
.screen {
mut scr := screen.new(reset: false) or { panic("can't get screen") }
return scr.exists(name)
}
.systemd {
// console.print_debug("exists sm systemd ${name}")
mut systemdfactory := systemd.new()!
return systemdfactory.exists(name)
}
.zinit {
// console.print_debug("exists sm zinit check ${name}")
mut zinitfactory := zinit.new()!
zinitfactory.load()!
return zinitfactory.exists(name)
}
else {
panic('to implement. startup manager only support screen & systemd for now: ${sm.cat}')
}
}
}
// list all services as known to the startup manager
pub fn (mut sm StartupManager) list() ![]string {
match sm.cat {
.screen {
// mut scr := screen.new(reset: false) or { panic("can't get screen") }
panic('implement')
}
.systemd {
mut systemdfactory := systemd.new()!
return systemdfactory.names()
}
.zinit {
mut zinitfactory := zinit.new()!
return zinitfactory.names()
}
else {
panic('to implement. startup manager only support screen & systemd for now: ${sm.cat}')
}
}
}
// THIS IS PROBABLY PART OF OTHER MODULE NOW
// pub struct SecretArgs {
// pub mut:
// name string @[required]
// cat SecretType
// }
// pub enum SecretType {
// normal
// }
// // creates a secret if it doesn exist yet
// pub fn (mut sm StartupManager) secret(args SecretArgs) !string {
// if !(sm.exists(args.name)) {
// return error("can't find screen with name ${args.name}, for secret")
// }
// key := 'secrets:startup:${args.name}'
// mut redis := redisclient.core_get()!
// mut secret := redis.get(key)!
// if secret.len == 0 {
// secret = rand.hex(16)
// redis.set(key, secret)!
// }
// return secret
// }

View File

@@ -1,34 +0,0 @@
module startupmanager
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.osal.screen
import freeflowuniverse.herolib.osal.systemd
const process_name = 'testprocess'
pub fn testsuite_begin() ! {
mut sm := get()!
if sm.exists(process_name)! {
sm.stop(process_name)!
}
}
pub fn testsuite_end() ! {
mut sm := get()!
if sm.exists(process_name)! {
sm.stop(process_name)!
}
}
// remove from the startup manager
pub fn test_status() ! {
mut sm := get()!
sm.start(
name: process_name
cmd: 'redis-server'
)!
status := sm.status(process_name)!
assert status == .active
}