MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

jina specs

Browse Source
This commit is contained in:
despiegk
2025-03-11 11:23:58 +01:00
parent 3da895083b
commit b1453e3580
20 changed files with 565 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
aiprompts/ai_core/osal_os_system_tools.md
View File

@@ -3,7 +3,7 @@
import as
```vlang
import freeflowuniverse.herolib.osal
import freeflowuniverse.osal
osal.ping...
@@ -70,14 +70,14 @@ mut pm:=process.processmap_get()?
info returns like:
```json
}, freeflowuniverse.herolib.process.ProcessInfo{
}, freeflowuniverse.process.ProcessInfo{
cpu_perc: 0
mem_perc: 0
cmd: 'mc'
pid: 84455
ppid: 84467
rss: 3168
}, freeflowuniverse.herolib.process.ProcessInfo{
}, freeflowuniverse.process.ProcessInfo{
cpu_perc: 0
mem_perc: 0
cmd: 'zsh -Z -g'

14
aiprompts/env.md Normal file
View File

@@ -0,0 +1,14 @@
## Environment Variables
```v
import freeflowuniverse.herolib.osal
// Get environment variable
value := osal.env_get('PATH')!
// Set environment variable
osal.env_set('MY_VAR', 'value')!
// Check if environment variable exists
exists := osal.env_exists('MY_VAR')
```

69
examples/aiexamples/openai_chat_completion.vsh Executable file
View File

@@ -0,0 +1,69 @@
#!/usr/bin/env -S v -n -w -gc none -cc tcc -d use_openssl -enable-globals run
import freeflowuniverse.herolib.clients.jina
import freeflowuniverse.herolib.osal
import os
// Example of using the Jina client
fn main() {
// Set environment variable for testing
// In production, you would set this in your environment
// osal.env_set(key: 'JINAKEY', value: 'your-api-key')
// Check if JINAKEY environment variable exists
if !osal.env_exists('JINAKEY') {
println('JINAKEY environment variable not set. Please set it before running this example.')
exit(1)
}
// Create a Jina client instance
mut client := jina.get(name: 'default')!
println('Jina client initialized successfully.')
// Example: Create embeddings
model := 'jina-embeddings-v3'
texts := ['Hello, world!', 'How are you doing?']
println('Creating embeddings for texts: ${texts}')
result := client.create_embeddings(texts, model, 'retrieval.query')!
println('Embeddings created successfully.')
println('Model: ${result['model']}')
println('Data count: ${result['data'].arr().len}')
// Example: List classifiers
println('\nListing classifiers:')
classifiers := client.list_classifiers() or {
println('Failed to list classifiers: ${err}')
return
}
println('Classifiers retrieved successfully.')
// Example: Create a classifier
println('\nTraining a classifier:')
examples := [
jina.TrainingExample{
text: 'This movie was great!'
label: 'positive'
},
jina.TrainingExample{
text: 'I did not like this movie.'
label: 'negative'
},
jina.TrainingExample{
text: 'The movie was okay.'
label: 'neutral'
}
]
training_result := client.train(examples, model, 'private') or {
println('Failed to train classifier: ${err}')
return
}
println('Classifier trained successfully.')
println('Classifier ID: ${training_result['classifier_id']}')
}

0
examples/aitest/atest.vsh
View File

86
examples/aitest/dir_listing.vsh
View File

@@ -1,86 +0,0 @@
#!/usr/bin/env -S v -n -w -gc none -cc tcc -d use_openssl -enable-globals run
import os
import freeflowuniverse.herolib.core.pathlib
// Helper function to format file sizes
fn format_size(size i64) string {
if size < 1024 {
return '${size} B'
} else if size < 1024 * 1024 {
kb := f64(size) / 1024.0
return '${kb:.1f} KB'
} else if size < 1024 * 1024 * 1024 {
mb := f64(size) / (1024.0 * 1024.0)
return '${mb:.1f} MB'
} else {
gb := f64(size) / (1024.0 * 1024.0 * 1024.0)
return '${gb:.1f} GB'
}
}
// Set parameters directly in the script
// Change these values as needed
target_dir := '/tmp' // Current directory by default
show_hidden := false // Set to true to show hidden files
recursive := false // Set to true for recursive listing
// Create a Path object for the target directory
mut path := pathlib.get(target_dir)
// Ensure the directory exists and is a directory
if path.exist == .no {
eprintln('Error: Directory "${target_dir}" does not exist')
exit(1)
}
if path.cat != .dir && path.cat != .linkdir {
eprintln('Error: "${target_dir}" is not a directory')
exit(1)
}
// Main execution
println('Listing contents of: ${path.absolute()}')
println('----------------------------')
// Define list arguments
mut list_args := pathlib.ListArgs{
recursive: recursive,
ignoredefault: !show_hidden
}
// Use pathlib to list the directory contents
mut list_result := path.list(list_args) or {
eprintln('Error listing directory: ${err}')
exit(1)
}
// Print each file/directory
for p in list_result.paths {
// Skip the root directory itself
if p.path == path.path {
continue
}
// Calculate the level based on the path depth relative to the root
rel_path := p.path.replace(list_result.root, '')
level := rel_path.count('/') - if rel_path.starts_with('/') { 1 } else { 0 }
// Print indentation based on level
if level > 0 {
print(' '.repeat(level))
}
// Print file/directory info
name := p.name()
if p.cat == .dir || p.cat == .linkdir {
println('📁 ${name}/')
} else {
// Get file size
file_size := os.file_size(p.path)
println('📄 ${name} (${format_size(file_size)})')
}
}
println('----------------------------')
println('Done!')

8
lib/clients/jina/.heroscript Normal file
View File

@@ -0,0 +1,8 @@
!!hero_code.generate_client
name:'jina'
classname:'Jina'
singleton:0
default:1
hasconfig:1
reset:0

169
lib/clients/jina/jina_client.v Normal file
View File

@@ -0,0 +1,169 @@
module jina
import freeflowuniverse.herolib.core.httpconnection
import net.http
import x.json2
// Create embeddings for input texts
pub fn (mut j Jina) create_embeddings(input []string, model string, task string) !map[string]json2.Any {
req := httpconnection.Request{
method: .post
prefix: 'v1/embeddings'
dataformat: .json
data: json.encode({
'model': model
'input': input
'task': task
})
}
return j.http.get_json_dict(req)!
}
// Create embeddings with a TextDoc input
pub struct TextDoc {
pub:
id string
text string
}
pub fn (mut j Jina) create_embeddings_with_docs(input []TextDoc, model string, task string) !map[string]json2.Any {
req := httpconnection.Request{
method: .post
prefix: 'v1/embeddings'
dataformat: .json
data: json.encode({
'model': model
'input': input
'task': task
})
}
return j.http.get_json_dict(req)!
}
// Rerank documents based on a query
pub fn (mut j Jina) rerank(query string, documents []string, model string) !map[string]json2.Any {
req := httpconnection.Request{
method: .post
prefix: 'v1/rerank'
dataformat: .json
data: json.encode({
'model': model
'query': query
'documents': documents
})
}
return j.http.get_json_dict(req)!
}
// Classify input texts
pub fn (mut j Jina) classify(input []string, model string, labels []string) !map[string]json2.Any {
req := httpconnection.Request{
method: .post
prefix: 'v1/classify'
dataformat: .json
data: json.encode({
'model': model
'input': input
'labels': labels
})
}
return j.http.get_json_dict(req)!
}
// Train a classifier
pub struct TrainingExample {
pub:
text string
label string
}
pub fn (mut j Jina) train(examples []TrainingExample, model string, access string) !map[string]json2.Any {
// Convert examples to the format expected by the API
mut input := []map[string]string{}
for example in examples {
input << {
'text': example.text
'label': example.label
}
}
req := httpconnection.Request{
method: .post
prefix: 'v1/train'
dataformat: .json
data: json.encode({
'model': model
'input': input
'access': access
})
}
return j.http.get_json_dict(req)!
}
// List classifiers
pub fn (mut j Jina) list_classifiers() !map[string]json2.Any {
req := httpconnection.Request{
method: .get
prefix: 'v1/classifiers'
}
return j.http.get_json_dict(req)!
}
// Delete a classifier
pub fn (mut j Jina) delete_classifier(classifier_id string) !map[string]json2.Any {
req := httpconnection.Request{
method: .delete
prefix: 'v1/classifiers/${classifier_id}'
}
return j.http.get_json_dict(req)!
}
// Create multi-vector embeddings
pub fn (mut j Jina) create_multi_vector(input []string, model string) !map[string]json2.Any {
req := httpconnection.Request{
method: .post
prefix: 'v1/multi-vector'
dataformat: .json
data: json.encode({
'model': model
'input': input
})
}
return j.http.get_json_dict(req)!
}
// Start a bulk embedding job
pub fn (mut j Jina) start_bulk_embedding(file_path string, model string, email string) !string {
// This endpoint requires multipart/form-data which is not directly supported by the current HTTPConnection
// We need to implement a custom solution for this
return error('Bulk embedding is not implemented yet')
}
// Check if the API key is valid by making a simple request
pub fn (mut j Jina) check_auth() !bool {
req := httpconnection.Request{
method: .get
prefix: '/'
}
response := j.http.get(req) or {
return error('Failed to connect to Jina API: ${err}')
}
// If we get a response, the API key is valid
return true
}
// Helper function to check if environment variable exists
pub fn env_exists(key string) bool {
env := os.environ()
return key in env
}

112
lib/clients/jina/jina_factory_.v Normal file
View File

@@ -0,0 +1,112 @@
module jina
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui.console
__global (
jina_global map[string]&Jina
jina_default string
)
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string
}
fn args_get(args_ ArgsGet) ArgsGet {
mut args := args_
if args.name == '' {
args.name = 'default'
}
return args
}
pub fn get(args_ ArgsGet) !&Jina {
mut context := base.context()!
mut args := args_get(args_)
mut obj := Jina{}
if args.name !in jina_global {
if !exists(args)! {
set(obj)!
} else {
heroscript := context.hero_config_get('jina', args.name)!
mut obj_ := heroscript_loads(heroscript)!
set_in_mem(obj_)!
}
}
return jina_global[args.name] or {
println(jina_global)
// bug if we get here because should be in globals
panic('could not get config for jina with name, is bug:${args.name}')
}
}
// register the config for the future
pub fn set(o Jina) ! {
set_in_mem(o)!
mut context := base.context()!
heroscript := heroscript_dumps(o)!
context.hero_config_set('jina', o.name, heroscript)!
}
// does the config exists?
pub fn exists(args_ ArgsGet) !bool {
mut context := base.context()!
mut args := args_get(args_)
return context.hero_config_exists('jina', args.name)
}
pub fn delete(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
context.hero_config_delete('jina', args.name)!
if args.name in jina_global {
// del jina_global[args.name]
}
}
// only sets in mem, does not set as config
fn set_in_mem(o Jina) ! {
mut o2 := obj_init(o)!
jina_global[o.name] = &o2
jina_default = o.name
}
@[params]
pub struct PlayArgs {
pub mut:
heroscript string // if filled in then plbook will be made out of it
plbook ?playbook.PlayBook
reset bool
}
pub fn play(args_ PlayArgs) ! {
mut args := args_
mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
mut install_actions := plbook.find(filter: 'jina.configure')!
if install_actions.len > 0 {
for install_action in install_actions {
heroscript := install_action.heroscript()
mut obj2 := heroscript_loads(heroscript)!
set(obj2)!
}
}
}
// switch instance to be used for jina
pub fn switch(name string) {
jina_default = name
}
// helpers
@[params]
pub struct DefaultConfigArgs {
instance string = 'default'
}

62
lib/clients/jina/jina_model.v Normal file
View File

@@ -0,0 +1,62 @@
module jina
import freeflowuniverse.herolib.data.paramsparser
import freeflowuniverse.herolib.data.encoderhero
import freeflowuniverse.herolib.core.httpconnection
import freeflowuniverse.herolib.osal
import os
pub const version = '0.0.0'
const singleton = false
const default = true
const api_base_url = 'https://api.jina.ai'
const env_key = 'JINAKEY'
// THIS THE THE SOURCE OF THE INFORMATION OF THIS FILE, HERE WE HAVE THE CONFIG OBJECT CONFIGURED AND MODELLED
@[heap]
pub struct Jina {
pub mut:
name string = 'default'
secret string
base_url string = api_base_url
http httpconnection.HTTPConnection @[str: skip]
}
// your checking & initialization code if needed
fn obj_init(mycfg_ Jina) !Jina {
mut mycfg := mycfg_
// Get API key from environment variable if not set
if mycfg.secret == '' {
if osal.env_exists(env_key) {
mycfg.secret = osal.env_get(env_key) or {
return error('Failed to get API key from environment variable ${env_key}: ${err}')
}
} else {
return error('Jina API key not provided and ${env_key} environment variable not set')
}
}
// Initialize HTTP connection
mycfg.http = httpconnection.HTTPConnection{
base_url: mycfg.base_url
default_header: http.new_header(
key: .authorization
value: 'Bearer ${mycfg.secret}'
)
}
return mycfg
}
/////////////NORMALLY NO NEED TO TOUCH
pub fn heroscript_dumps(obj Jina) !string {
return encoderhero.encode[Jina](obj)!
}
pub fn heroscript_loads(heroscript string) !Jina {
mut obj := encoderhero.decode[Jina](heroscript)!
return obj
}

40
lib/clients/jina/py_specs/classify.py Normal file
View File

@@ -0,0 +1,40 @@
import json
import requests
url = "https://api.jina.ai/v1/classify"
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer jina_275aefb6495643408d4c499fce548080w5rYjijHfHVBi_vtAqNY6LBk-woz"
}
data = {
"model": "jina-clip-v2",
"input": [
{
"text": "A sleek smartphone with a high-resolution display and multiple camera lenses"
},
{
"text": "Fresh sushi rolls served on a wooden board with wasabi and ginger"
},
{
"image": "https://picsum.photos/id/11/367/267"
},
{
"image": "https://picsum.photos/id/22/367/267"
},
{
"text": "Vibrant autumn leaves in a dense forest with sunlight filtering through"
},
{
"image": "https://picsum.photos/id/8/367/267"
}
],
"labels": [
"Technology and Gadgets",
"Food and Dining",
"Nature and Outdoors",
"Urban and Architecture"
]
}
response = requests.post(url, headers=headers, data=json.dumps(data))
print(response.json())

24
lib/clients/jina/py_specs/embeddings.py Normal file
View File

@@ -0,0 +1,24 @@
import requests
import json
url = "https://api.jina.ai/v1/embeddings"
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer jina_275aefb6495643408d4c499fce548080w5rYjijHfHVBi_vtAqNY6LBk-woz"
}
data = {
"model": "jina-embeddings-v3",
"task": "text-matching",
"late_chunking": False,
"dimensions": "1024",
"embedding_type": "ubinary",
"input": [
"Organic skincare for sensitive skin with aloe vera and chamomile: Imagine the soothing embrace of nature with our organic skincare range, crafted specifically for sensitive skin. Infused with the calming properties of aloe vera and chamomile, each product provides gentle nourishment and protection. Say goodbye to irritation and hello to a glowing, healthy complexion.",
"Bio-Hautpflege für empfindliche Haut mit Aloe Vera und Kamille: Erleben Sie die wohltuende Wirkung unserer Bio-Hautpflege, speziell für empfindliche Haut entwickelt. Mit den beruhigenden Eigenschaften von Aloe Vera und Kamille pflegen und schützen unsere Produkte Ihre Haut auf natürliche Weise. Verabschieden Sie sich von Hautirritationen und genießen Sie einen strahlenden Teint.",
]
}
response = requests.post(url, headers=headers, data=json.dumps(data))
print(response.json())

18
lib/clients/jina/py_specs/llmserp.py Normal file
View File

@@ -0,0 +1,18 @@
import json
import requests
url = "https://llm-serp.jina.ai"
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer jina_275aefb6495643408d4c499fce548080w5rYjijHfHVBi_vtAqNY6LBk-woz"
}
data = {
"q": "jina ai",
"gl": "US",
"hl": "en",
"num": 10,
"page": 1
}
response = requests.post(url, headers=headers, data=json.dumps(data))
print(response.json())

16
lib/clients/jina/py_specs/readerlm.py Normal file
View File

@@ -0,0 +1,16 @@
import requests
url = "https://r.jina.ai/https://www.threefold.io/what/"
headers = {
"Accept": "text/event-stream",
"Authorization": "Bearer jina_275aefb6495643408d4c499fce548080w5rYjijHfHVBi_vtAqNY6LBk-woz",
"X-Base": "final",
"X-Respond-With": "readerlm-v2",
"X-Return-Format": "markdown",
"X-With-Iframe": "true",
"X-With-Shadow-Dom": "true"
}
response = requests.get(url, headers=headers)
print(response.text)

30
lib/clients/jina/readme.md Normal file
View File

@@ -0,0 +1,30 @@
# jina
To get started
```vlang
import freeflowuniverse.herolib.clients. jina
mut client:= jina.get()!
client...
```
## example heroscript
```hero
!!jina.configure
secret: '...'
host: 'localhost'
port: 8888
```

25
lib/clients/openai/_archive/actions.v
View File

@@ -1,25 +0,0 @@
module openai
import freeflowuniverse.herolib.core.playbook
// run heroscript starting from path, text or giturl
//```
// !!OpenAIclient.define
// name:'default'
// openaikey: ''
// description:'...'
//```
pub fn heroplay(mut plbook playbook.PlayBook) ! {
for mut action in plbook.find(filter: 'openaiclient.define')! {
mut p := action.params
instance := p.get_default('instance', 'default')!
// cfg.keyname = p.get('keyname')!
mut cl := get(instance,
openaikey: p.get('openaikey')!
description: p.get_default('description', '')!
)!
cl.config_save()!
}
}
//>TODO: this needs to be extended to chats, ...

64
lib/clients/openai/_archive/factory.v
View File

@@ -1,64 +0,0 @@
module openai
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui as gui
import freeflowuniverse.herolib.core.httpconnection
// import freeflowuniverse.herolib.ui.console
pub struct OpenAIClient[T] {
base.BaseConfig[T]
pub mut:
connection &httpconnection.HTTPConnection
}
@[params]
pub struct Config {
pub mut:
openaikey string @[secret]
description string
}
pub fn get(instance string, cfg Config) !OpenAIClient[Config] {
mut self := OpenAIClient[Config]{
connection: &httpconnection.HTTPConnection{}
}
if cfg.openaikey.len > 0 {
// first the type of the instance, then name of instance, then action
self.init('openaiclient', instance, .set, cfg)!
} else {
self.init('openaiclient', instance, .get)!
}
mut conn := httpconnection.new(
name: 'openai'
url: 'https://api.openai.com/v1/'
)!
conn.default_header.add(.authorization, 'Bearer ${self.config()!.openaikey}')
// req.add_custom_header('x-disable-pagination', 'True') !
self.connection = conn
return self
}
// get a new OpenAI client, will create if it doesn't exist or ask for new configuration
pub fn configure(instance_ string) ! {
mut cfg := Config{}
mut ui := gui.new()!
mut instance := instance_
if instance == '' {
instance = ui.ask_question(
question: 'name for Dagu client'
default: instance
)!
}
cfg.openaikey = ui.ask_question(
question: '\nPlease specify your openai secret (instance:${instance}).'
)!
get(instance, cfg)!
}

27
lib/clients/openai/_archive/openai_factory_.v
View File

@@ -1,27 +0,0 @@
module openai
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui.console
__global (
openai_global map[string]&OpenAI
openai_default string
)
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string
}
pub fn get(args_ ArgsGet) !&OpenAI {
return &OpenAI{}
}
// switch instance to be used for openai
pub fn switch(name string) {
openai_default = name
}

49
lib/clients/openai/_archive/openai_model.v
View File

@@ -1,49 +0,0 @@
module openai
import freeflowuniverse.herolib.data.paramsparser
import os
import freeflowuniverse.herolib.core.httpconnection
pub const version = '0.0.0'
const singleton = false
const default = true
// THIS THE THE SOURCE OF THE INFORMATION OF THIS FILE, HERE WE HAVE THE CONFIG OBJECT CONFIGURED AND MODELLED
@[heap]
pub struct OpenAI {
pub mut:
name string = 'default'
mail_from string
mail_password string @[secret]
mail_port int
mail_server string
mail_username string
}
fn obj_init(obj_ OpenAI) !OpenAI {
// never call get here, only thing we can do here is work on object itself
mut obj := obj_
panic('implement')
return obj
}
pub fn (mut client OpenAI) connection() !&httpconnection.HTTPConnection {
mut c := client.conn or {
mut c2 := httpconnection.new(
name: 'openrouterclient_${client.name}'
url: 'https://openrouter.ai/api/v1/chat/completions'
cache: false
retry: 0
)!
c2
}
// see https://modules.vlang.io/net.http.html#CommonHeader
// -H "Authorization: Bearer $OPENROUTER_API_KEY" \
c.default_header.set(.authorization, 'Bearer ${client.openaikey}')
c.default_header.add_custom('HTTP-Referer', client.your_site_url)!
c.default_header.add_custom('X-Title', client.your_site_name)!
client.conn = c
return c
}

105
lib/clients/openai/openai_factory_.v
View File

@@ -1,105 +0,0 @@
module openai
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
__global (
openai_global map[string]&OpenAI
openai_default string
)
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string = 'default'
}
fn args_get(args_ ArgsGet) ArgsGet {
mut args := args_
if args.name == '' {
args.name = openai_default
}
if args.name == '' {
args.name = 'default'
}
return args
}
pub fn get(args_ ArgsGet) !&OpenAI {
mut args := args_get(args_)
if args.name !in openai_global {
if !config_exists() {
if default {
config_save()!
}
}
config_load()!
}
return openai_global[args.name] or {
println(openai_global)
panic('bug in get from factory: ')
}
}
fn config_exists(args_ ArgsGet) bool {
mut args := args_get(args_)
mut context := base.context() or { panic('bug') }
return context.hero_config_exists('openai', args.name)
}
fn config_load(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
mut heroscript := context.hero_config_get('openai', args.name)!
play(heroscript: heroscript)!
}
fn config_save(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
context.hero_config_set('openai', args.name, heroscript_default()!)!
}
fn set(o OpenAI) ! {
mut o2 := obj_init(o)!
openai_global['default'] = &o2
}
@[params]
pub struct PlayArgs {
pub mut:
name string = 'default'
heroscript string // if filled in then plbook will be made out of it
plbook ?playbook.PlayBook
reset bool
start bool
stop bool
restart bool
delete bool
configure bool // make sure there is at least one installed
}
pub fn play(args_ PlayArgs) ! {
mut args := args_
if args.heroscript == '' {
args.heroscript = heroscript_default()!
}
mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
mut install_actions := plbook.find(filter: 'openai.configure')!
if install_actions.len > 0 {
for install_action in install_actions {
mut p := install_action.params
cfg_play(p)!
}
}
}
// switch instance to be used for openai
pub fn switch(name string) {
openai_default = name
}

0
lib/clients/openai/openai_model.v → lib/clients/openai/openai_model_.v
View File