MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

Merge branch 'development' of https://github.com/freeflowuniverse/herolib into development

Browse Source
This commit is contained in:
despiegk
2025-01-14 06:26:13 +01:00
parent 9c5fc9e65c dac3b61116
commit e1c93e8f98
26 changed files with 667 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/clients/openai/.heroscript
View File

@@ -1,7 +1,8 @@
!!hero_code.generate_client
name: "openai"
classname: "OpenAI"
hasconfig: false
singleton: false
default: true
title: ""
name:'openai'
classname:'OpenAI'
singleton:0
default:1
hasconfig:1
reset:0

89
lib/clients/openai/openai_factory_.v
View File

@@ -2,7 +2,6 @@ module openai
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui.console
__global (
openai_global map[string]&OpenAI
@@ -14,35 +13,93 @@ __global (
@[params]
pub struct ArgsGet {
pub mut:
name string
name string = 'default'
}
fn args_get(args_ ArgsGet) ArgsGet {
mut model := args_
if model.name == '' {
model.name = openai_default
mut args := args_
if args.name == '' {
args.name = openai_default
}
if model.name == '' {
model.name = 'default'
if args.name == '' {
args.name = 'default'
}
return model
return args
}
pub fn get(args_ ArgsGet) !&OpenAI {
mut args := args_get(args_)
if args.name !in openai_global {
if args.name == 'default' {
if !config_exists(args) {
if default {
mut context := base.context() or { panic('bug') }
context.hero_config_set('openai', model.name, heroscript_default()!)!
}
if !config_exists() {
if default {
config_save()!
}
load(args)!
}
config_load()!
}
return openai_global[args.name] or {
println(openai_global)
panic('could not get config for ${args.name} with name:${model.name}')
panic('bug in get from factory: ')
}
}
fn config_exists(args_ ArgsGet) bool {
mut args := args_get(args_)
mut context := base.context() or { panic('bug') }
return context.hero_config_exists('openai', args.name)
}
fn config_load(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
mut heroscript := context.hero_config_get('openai', args.name)!
play(heroscript: heroscript)!
}
fn config_save(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
context.hero_config_set('openai', args.name, heroscript_default()!)!
}
fn set(o OpenAI) ! {
mut o2 := obj_init(o)!
openai_global['default'] = &o2
}
@[params]
pub struct PlayArgs {
pub mut:
name string = 'default'
heroscript string // if filled in then plbook will be made out of it
plbook ?playbook.PlayBook
reset bool
start bool
stop bool
restart bool
delete bool
configure bool // make sure there is at least one installed
}
pub fn play(args_ PlayArgs) ! {
mut args := args_
if args.heroscript == '' {
args.heroscript = heroscript_default()!
}
mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
mut install_actions := plbook.find(filter: 'openai.configure')!
if install_actions.len > 0 {
for install_action in install_actions {
mut p := install_action.params
cfg_play(p)!
}
}
}
// switch instance to be used for openai
pub fn switch(name string) {
openai_default = name
}

4
lib/clients/openai/openai_model.v
View File

@@ -2,6 +2,7 @@ module openai
import freeflowuniverse.herolib.data.paramsparser
import freeflowuniverse.herolib.core.httpconnection
import os
pub const version = '1.14.3'
const singleton = false
@@ -12,7 +13,7 @@ pub fn heroscript_default() !string {
heroscript := "
!!openai.configure
name:'openai'
key: 'YOUR_API_KEY'
api_key: ${os.getenv('OPENAI_API_KEY')}
"
return heroscript
@@ -49,6 +50,7 @@ pub fn (mut client OpenAI) connection() !&httpconnection.HTTPConnection {
name: 'openaiconnection_${client.name}'
url: 'https://api.openai.com/v1'
cache: false
retry: 20
)!
c2
}

13
lib/clients/rclone/.heroscript
View File

@@ -1,7 +1,8 @@
!!hero_code.generate_client
name: "rclone"
classname: "RCloneClient"
hasconfig: true
singleton: true
default: true
title: ""
name:'rclone'
classname:'RCloneClient'
singleton:1
default:1
hasconfig:1
reset:0

97
lib/clients/rclone/rclone_factory_.v
View File

@@ -2,8 +2,6 @@ module rclone
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.data.encoderhero
__global (
rclone_global map[string]&RCloneClient
@@ -12,59 +10,96 @@ __global (
/////////FACTORY
// set the model in mem and the config on the filesystem
pub fn set(o RCloneClient) ! {
mut o2 := obj_init(o)!
rclone_global[o.name] = &o2
rclone_default = o.name
@[params]
pub struct ArgsGet {
pub mut:
name string = 'default'
}
// check we find the config on the filesystem
pub fn exists(args_ ArgsGet) bool {
mut model := args_get(args_)
fn args_get(args_ ArgsGet) ArgsGet {
mut args := args_
if args.name == '' {
args.name = rclone_default
}
if args.name == '' {
args.name = 'default'
}
return args
}
pub fn get(args_ ArgsGet) !&RCloneClient {
mut args := args_get(args_)
if args.name !in rclone_global {
if !config_exists() {
if default {
config_save()!
}
}
config_load()!
}
return rclone_global[args.name] or {
println(rclone_global)
panic('bug in get from factory: ')
}
}
fn config_exists(args_ ArgsGet) bool {
mut args := args_get(args_)
mut context := base.context() or { panic('bug') }
return context.hero_config_exists('rclone', model.name)
return context.hero_config_exists('rclone', args.name)
}
// load the config error if it doesn't exist
pub fn load(args_ ArgsGet) ! {
mut model := args_get(args_)
fn config_load(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
mut heroscript := context.hero_config_get('rclone', model.name)!
mut heroscript := context.hero_config_get('rclone', args.name)!
play(heroscript: heroscript)!
}
// save the config to the filesystem in the context
pub fn save(o RCloneClient) ! {
fn config_save(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
heroscript := encoderhero.encode[RCloneClient](o)!
context.hero_config_set('rclone', model.name, heroscript)!
context.hero_config_set('rclone', args.name, heroscript_default()!)!
}
fn set(o RCloneClient) ! {
mut o2 := obj_init(o)!
rclone_global['default'] = &o2
}
@[params]
pub struct PlayArgs {
pub mut:
name string = 'default'
heroscript string // if filled in then plbook will be made out of it
plbook ?playbook.PlayBook
reset bool
start bool
stop bool
restart bool
delete bool
configure bool // make sure there is at least one installed
}
pub fn play(args_ PlayArgs) ! {
mut model := args_
mut args := args_
if model.heroscript == '' {
model.heroscript = heroscript_default()!
if args.heroscript == '' {
args.heroscript = heroscript_default()!
}
mut plbook := model.plbook or { playbook.new(text: model.heroscript)! }
mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
mut configure_actions := plbook.find(filter: 'rclone.configure')!
if configure_actions.len > 0 {
for config_action in configure_actions {
mut p := config_action.params
mycfg := cfg_play(p)!
console.print_debug('install action rclone.configure\n${mycfg}')
set(mycfg)!
save(mycfg)!
mut install_actions := plbook.find(filter: 'rclone.configure')!
if install_actions.len > 0 {
for install_action in install_actions {
mut p := install_action.params
cfg_play(p)!
}
}
}
// switch instance to be used for rclone
pub fn switch(name string) {
rclone_default = name
}

4
lib/core/base/context.v
View File

@@ -70,8 +70,8 @@ pub fn (mut self Context) redis() !&redisclient.Redis {
// make sure we are on the right db
r.selectdb(int(self.config.id))!
}
self.redis_ = &r
&r
self.redis_ = r
r
}
return r2

23
lib/core/logger/log_test.v
View File

@@ -81,16 +81,25 @@ fn test_logger() {
create: false
)!
println('/tmp/testlogs/${files[0]}')
content := file.read()!.trim_space()
items := logger.search()!
assert items.len == 6 // still wrong: TODO
items_stdout := logger.search(
timestamp_from: ourtime.new('2022-11-1 20:14:35')!
timestamp_to: ourtime.new('2025-11-1 20:14:35')!
logtype: .stdout
)!
assert items_stdout.len == 2
items_error := logger.search(
timestamp_from: ourtime.new('2022-11-1 20:14:35')!
timestamp_to: ourtime.new('2025-11-1 20:14:35')!
logtype: .error
)!
assert items_error.len == 4
}
fn testsuite_end() {
if os.exists('/tmp/testlogs') {
os.rmdir_all('/tmp/testlogs')!
}
// if os.exists('/tmp/testlogs') {
// os.rmdir_all('/tmp/testlogs')!
// }
}

17
lib/core/logger/search.v
View File

@@ -30,7 +30,6 @@ pub fn (mut l Logger) search(args_ SearchArgs) ![]LogItem {
// Get time range
from_time := timestamp_from.unix()
to_time := timestamp_to.unix()
if from_time > to_time {
return error('from_time cannot be after to_time: ${from_time} < ${to_time}')
}
@@ -82,20 +81,30 @@ pub fn (mut l Logger) search(args_ SearchArgs) ![]LogItem {
continue
}
if collecting && line.len > 14 && line[13] == `-` {
process(mut result, current_item, current_time, args, from_time, to_time)!
collecting = false
}
// Parse log line
is_error := line.starts_with('E')
if !collecting {
// Start new item
current_item = LogItem{
timestamp: current_time
cat: line_trim[2..12].trim_space()
log: line_trim[15..].trim_space()
cat: line[2..12].trim_space()
log: line[15..].trim_space()
logtype: if is_error { .error } else { .stdout }
}
// println('new current item: ${current_item}')
collecting = true
} else {
// Continuation line
current_item.log += '\n' + line_trim[15..]
if line_trim.len < 16 {
current_item.log += '\n'
} else {
current_item.log += '\n' + line[15..]
}
}
}

2
lib/core/playbook/factory.v
View File

@@ -29,7 +29,7 @@ pub mut:
pub fn new(args_ PlayBookNewArgs) !PlayBook {
mut args := args_
mut c := base.context()!
mut c := base.context() or { return error('failed to get context: ${err}') }
mut s := c.session_new()!

2
lib/core/playcmds/play_dagu_test.v
View File

@@ -27,5 +27,5 @@ const dagu_script = "
fn test_play_dagu() ! {
mut plbook := playbook.new(text: dagu_script)!
play_dagu(mut plbook)!
panic('s')
// panic('s')
}

135
lib/core/playcmds/play_mdbook.v
View File

@@ -1,135 +0,0 @@
module playcmds
import freeflowuniverse.herolib.web.mdbook
import freeflowuniverse.herolib.data.doctree
import freeflowuniverse.herolib.core.playbook
import os
pub fn play_mdbook(mut plbook playbook.PlayBook) ! {
mut buildroot := '${os.home_dir()}/hero/var/mdbuild'
mut publishroot := '${os.home_dir()}/hero/www/info'
mut coderoot := ''
// mut install := false
mut reset := false
mut pull := false
// check if any actions for doctree, if not then nothing to do here
// dtactions := plbook.find(filter: 'doctree.')!
// if dtactions.len == 0 {
// console.print_debug("can't find doctree.add statements, nothing to do")
// return
// }
mut config_actions := plbook.find(filter: 'books:configure')!
if config_actions.len > 1 {
return error('can only have 1 config action for books')
} else if config_actions.len == 1 {
mut p := config_actions[0].params
if p.exists('buildroot') {
buildroot = p.get('buildroot')!
}
if p.exists('coderoot') {
coderoot = p.get('coderoot')!
}
if p.exists('publishroot') {
publishroot = p.get('publishroot')!
}
if p.exists('reset') {
reset = p.get_default_false('reset')
}
config_actions[0].done = true
}
mut trees := map[string]&doctree.Tree{}
for mut action in plbook.find(filter: 'doctree:new')! {
mut p := action.params
name := p.get('name')!
fail_on_error := p.get_default_false('fail_on_error')
if name in trees {
return error('tree with name ${name} already exists')
}
tree := doctree.new(name: name, fail_on_error: fail_on_error)!
trees[name] = tree
}
for mut action in plbook.find(filter: 'doctree:add')! {
mut p := action.params
url := p.get_default('url', '')!
path := p.get_default('path', '')!
name := p.get_default('name', '')!
if trees.len == 0 {
return error('no tree found')
}
mut tree := if name != '' {
trees[name] or { return error('tree ${name} not found') }
} else {
trees.values()[0]
}
tree.scan(
path: path
git_url: url
git_reset: reset
git_root: coderoot
git_pull: pull
)!
action.done = true
}
for mut action in plbook.find(filter: 'doctree:export')! {
mut p := action.params
build_path := p.get('path')!
toreplace := p.get_default('replace', '')!
reset2 := p.get_default_false('reset')
name := p.get('name')!
mut tree := trees[name] or { return error('tree: ${name} not found') }
tree.export(
destination: build_path
reset: reset2
toreplace: toreplace
)!
action.done = true
}
for mut action in plbook.find(filter: 'mdbook:export')! {
mut p := action.params
name := p.get('name')!
summary_url := p.get_default('summary_url', '')!
summary_path := p.get_default('summary_path', '')!
title := p.get_default('title', name)!
publish_path := p.get_default('publish_path', '${publishroot}/${name}')!
build_path := p.get_default('build_path', '${buildroot}/${name}')!
printbook := p.get_default_false('printbook')
foldlevel := p.get_int_default('foldlevel', 0)!
production := p.get_default_false('production')
reset3 := p.get_default_true('reset')
collections := p.get_list_default('collections', [])!
if summary_url == '' && summary_path == '' {
return error('Both summary url and path cannot be empty at the same time')
}
mut mdbooks := mdbook.get()!
mdbooks.path_build = buildroot
mdbooks.path_publish = publishroot
mdbooks.generate(
name: name
title: title
summary_path: summary_path
publish_path: publish_path
build_path: build_path
printbook: printbook
foldlevel: foldlevel
production: production
collections: collections
)!
action.done = true
}
}

85
lib/core/playcmds/play_mdbook_test.v
View File

@@ -1,85 +0,0 @@
module playcmds
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.core.playcmds
import freeflowuniverse.herolib.core.pathlib
fn test_play_mdbook() {
mut summary_path := pathlib.get_file(path: '/tmp/mdbook_test/SUMMARY.md', create: true)!
summar_content := '
- [Page number 1](fruits/apple.md)
- [fruit intro](fruits/intro.md)
- [rpc page](rpc/tfchain.md)
- [vegies](test_vegetables/tomato.md)
'
summary_path.write(summar_content)!
mut p := pathlib.get_file(path: '/tmp/heroscript/do.hero', create: true)!
// script := "
// !!doctree.new
// name: 'tree1'
// !!doctree.add
// name: 'tree1'
// url:'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/actions'
// !!doctree.add
// name: 'tree1'
// url: 'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/fruits'
// !!doctree.add
// name: 'tree1'
// url: 'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/rpc'
// !!doctree.export
// name: 'tree1'
// path: '/tmp/export_tree1'
// !!doctree.new
// name: 'tree2'
// fail_on_error: true
// !!doctree.add
// name: 'tree2'
// url: 'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/vegetables'
// !!doctree.export
// name: 'tree2'
// path: '/tmp/export_tree2'
// !!mdbook.export
// title:'ThreeFold Technology'
// name:'tech'
// summary_path:'${summary_path.path}'
// collections:'/tmp/export_tree1,/tmp/export_tree2'
// dest: '/tmp/mdbook_export'
// production:0 //means we put it in summary
// "
s2 := "
!!doctree.new
name: 'info_tfgrid'
fail_on_error: false
!!doctree.add
name:'info_tfgrid'
url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/main/collections'
!!doctree.export
name:'info_tfgrid'
path:'~/hero/var/collections/info_tfgrid'
!!mdbook.export
title:'ThreeFold Technology'
name:'tech'
summary_url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/development/books/tech/SUMMARY.md'
collections:'~/hero/var/collections/info_tfgrid'
production:0 //means we put it in summary
"
p.write(s2)!
mut plbook := playbook.new(path: '/tmp/heroscript')!
playcmds.play_mdbook(mut plbook)!
}

8
lib/core/playcmds/play_publisher.v Normal file
View File

@@ -0,0 +1,8 @@
module playcmds
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.hero.publishing
pub fn play_publisher(mut plbook playbook.PlayBook) ! {
publishing.play(mut plbook)!
}

34
lib/core/playcmds/play_publisher_test.v Normal file
View File

@@ -0,0 +1,34 @@
module playcmds
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.core.playcmds
import freeflowuniverse.herolib.core.pathlib
import os
fn test_play_publisher() {
mut p := pathlib.get_file(path: '/tmp/heroscript/do.hero', create: true)!
s2 := "
!!publisher.new_collection
url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/main/collections'
reset: false
pull: true
!!book.define
name:'info_tfgrid'
summary_url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/development/books/tech/SUMMARY.md'
title:'ThreeFold Technology'
collections: 'about,dashboard,farmers,library,partners_utilization,tech,p2p'
!!book.publish
name:'tech'
production: false
"
p.write(s2)!
mut plbook := playbook.new(path: '/tmp/heroscript')!
playcmds.play_publisher(mut plbook)!
}

20
lib/core/redisclient/factory.v
View File

@@ -3,11 +3,11 @@ module redisclient
// original code see https://github.com/patrickpissurno/vredis/blob/master/vredis_test.v
// credits see there as well (-:
import net
// import sync
import sync
// import strconv
__global (
redis_connections []Redis
redis_connections []&Redis
)
const default_read_timeout = net.infinite_timeout
@@ -18,15 +18,16 @@ pub:
addr string
mut:
socket net.TcpConn
mtx sync.RwMutex
}
// https://redis.io/topics/protocol
// examples:
// localhost:6379
// /tmp/redis-default.sock
pub fn new(addr string) !Redis {
// lock redis_connections {
for mut conn in redis_connections {
pub fn new(addr string) !&Redis {
// lock redis_cowritennections {
for conn in redis_connections {
if conn.addr == addr {
return conn
}
@@ -34,10 +35,13 @@ pub fn new(addr string) !Redis {
// means there is no connection yet
mut r := Redis{
addr: addr
mtx: sync.RwMutex{}
}
r.mtx.init()
r.socket_connect()!
redis_connections << r
return r
redis_connections << &r
return &r
//}
// panic("bug")
}
@@ -47,7 +51,7 @@ pub fn reset() ! {
for mut conn in redis_connections {
conn.disconnect()
}
redis_connections = []Redis{}
redis_connections = []&Redis{}
//}
}

2
lib/core/redisclient/redisclient_core.v
View File

@@ -18,7 +18,7 @@ pub fn get_redis_url(url string) !RedisURL {
}
}
pub fn core_get(url RedisURL) !Redis {
pub fn core_get(url RedisURL) !&Redis {
mut r := new('${url.address}:${url.port}')!
return r
}

18
lib/core/redisclient/redisclient_encode.v
View File

@@ -2,7 +2,7 @@ module redisclient
import freeflowuniverse.herolib.data.resp
pub fn (mut r Redis) get_response() !resp.RValue {
fn (mut r Redis) get_response() !resp.RValue {
line := r.read_line()!
if line.starts_with('-') {
@@ -63,7 +63,7 @@ pub fn (mut r Redis) get_response() !resp.RValue {
// TODO: needs to use the resp library
pub fn (mut r Redis) get_int() !int {
fn (mut r Redis) get_int() !int {
line := r.read_line()!
if line.starts_with(':') {
return line[1..].int()
@@ -72,7 +72,7 @@ pub fn (mut r Redis) get_int() !int {
}
}
pub fn (mut r Redis) get_list_int() ![]int {
fn (mut r Redis) get_list_int() ![]int {
line := r.read_line()!
mut res := []int{}
@@ -89,7 +89,7 @@ pub fn (mut r Redis) get_list_int() ![]int {
}
}
pub fn (mut r Redis) get_list_str() ![]string {
fn (mut r Redis) get_list_str() ![]string {
line := r.read_line()!
mut res := []string{}
@@ -106,7 +106,7 @@ pub fn (mut r Redis) get_list_str() ![]string {
}
}
pub fn (mut r Redis) get_string() !string {
fn (mut r Redis) get_string() !string {
line := r.read_line()!
if line.starts_with('+') {
// console.print_debug("getstring:'${line[1..]}'")
@@ -120,12 +120,12 @@ pub fn (mut r Redis) get_string() !string {
}
}
pub fn (mut r Redis) get_string_nil() !string {
fn (mut r Redis) get_string_nil() !string {
r2 := r.get_bytes_nil()!
return r2.bytestr()
}
pub fn (mut r Redis) get_bytes_nil() ![]u8 {
fn (mut r Redis) get_bytes_nil() ![]u8 {
line := r.read_line()!
if line.starts_with('+') {
return line[1..].bytes()
@@ -140,12 +140,12 @@ pub fn (mut r Redis) get_bytes_nil() ![]u8 {
}
}
pub fn (mut r Redis) get_bool() !bool {
fn (mut r Redis) get_bool() !bool {
i := r.get_int()!
return i == 1
}
pub fn (mut r Redis) get_bytes() ![]u8 {
fn (mut r Redis) get_bytes() ![]u8 {
line := r.read_line()!
if line.starts_with('$') {
return r.get_bytes_from_line(line)

4
lib/core/redisclient/redisclient_internal.v
View File

@@ -53,7 +53,7 @@ fn (mut r Redis) socket_check() ! {
}
}
pub fn (mut r Redis) read_line() !string {
fn (mut r Redis) read_line() !string {
return r.socket.read_line().trim_right('\r\n')
}
@@ -99,7 +99,7 @@ fn (mut r Redis) write_line(data []u8) ! {
}
// write resp value to the redis channel
pub fn (mut r Redis) write_rval(val resp.RValue) ! {
fn (mut r Redis) write_rval(val resp.RValue) ! {
r.write(val.encode())!
}

2
lib/core/redisclient/redisclient_sadd_test.v
View File

@@ -3,7 +3,7 @@ import freeflowuniverse.herolib.core.redisclient
fn setup() !&redisclient.Redis {
mut redis := redisclient.core_get()!
redis.selectdb(10) or { panic(err) }
return &redis
return redis
}
fn cleanup(mut redis redisclient.Redis) ! {

32
lib/core/redisclient/redisclient_send.v
View File

@@ -5,6 +5,10 @@ import freeflowuniverse.herolib.ui.console
// send list of strings, expect OK back
pub fn (mut r Redis) send_expect_ok(items []string) ! {
r.mtx.lock()
defer {
r.mtx.unlock()
}
r.write_cmds(items)!
res := r.get_string()!
if res != 'OK' {
@@ -15,23 +19,39 @@ pub fn (mut r Redis) send_expect_ok(items []string) ! {
// send list of strings, expect int back
pub fn (mut r Redis) send_expect_int(items []string) !int {
r.mtx.lock()
defer {
r.mtx.unlock()
}
r.write_cmds(items)!
return r.get_int()
}
pub fn (mut r Redis) send_expect_bool(items []string) !bool {
r.mtx.lock()
defer {
r.mtx.unlock()
}
r.write_cmds(items)!
return r.get_bool()
}
// send list of strings, expect string back
pub fn (mut r Redis) send_expect_str(items []string) !string {
r.mtx.lock()
defer {
r.mtx.unlock()
}
r.write_cmds(items)!
return r.get_string()
}
// send list of strings, expect string or nil back
pub fn (mut r Redis) send_expect_strnil(items []string) !string {
r.mtx.lock()
defer {
r.mtx.unlock()
}
r.write_cmds(items)!
d := r.get_string_nil()!
return d
@@ -39,16 +59,28 @@ pub fn (mut r Redis) send_expect_strnil(items []string) !string {
// send list of strings, expect list of strings back
pub fn (mut r Redis) send_expect_list_str(items []string) ![]string {
r.mtx.lock()
defer {
r.mtx.unlock()
}
r.write_cmds(items)!
return r.get_list_str()
}
pub fn (mut r Redis) send_expect_list_int(items []string) ![]int {
r.mtx.lock()
defer {
r.mtx.unlock()
}
r.write_cmds(items)!
return r.get_list_int()
}
pub fn (mut r Redis) send_expect_list(items []string) ![]resp.RValue {
r.mtx.lock()
defer {
r.mtx.unlock()
}
r.write_cmds(items)!
res := r.get_response()!
return resp.get_redis_array(res)

2
lib/core/redisclient/redisclient_test.v
View File

@@ -8,7 +8,7 @@ fn setup() !&redisclient.Redis {
mut redis := redisclient.core_get()!
// Select db 10 to be away from default one '0'
redis.selectdb(10) or { panic(err) }
return &redis
return redis
}
fn cleanup(mut redis redisclient.Redis) ! {

2
lib/develop/gittools/repository_cache.v
View File

@@ -3,7 +3,7 @@ module gittools
import json
import freeflowuniverse.herolib.core.redisclient
fn redis_get() redisclient.Redis {
fn redis_get() &redisclient.Redis {
mut redis_client := redisclient.core_get() or { panic(err) }
return redis_client
}

141
lib/hero/publishing/command.v Normal file
View File

@@ -0,0 +1,141 @@
module publishing
import cli { Command, Flag }
import freeflowuniverse.herolib.ui.console
// path string //if location on filessytem, if exists, this has prio on git_url
// git_url string // location of where the hero scripts are
// git_pull bool // means when getting new repo will pull even when repo is already there
// git_pullreset bool // means we will force a pull and reset old content
// coderoot string //the location of coderoot if its another one
pub fn cmd_publisher(pre_func fn (Command) !) Command {
mut cmd_publisher := Command{
name: 'publisher'
usage: '
## Manage your publications
example:
hero publisher -u https://git.ourworld.tf/ourworld_holding/info_ourworld/src/branch/develop/heroscript
If you do -gp it will pull newest book content from git and give error if there are local changes.
If you do -gr it will pull newest book content from git and overwrite local changes (careful).
'
description: 'create, edit, show mdbooks'
required_args: 0
execute: cmd_publisher_execute
pre_execute: pre_func
}
// cmd_run_add_flags(mut cmd_publisher)
cmd_publisher.add_flag(Flag{
flag: .string
name: 'name'
abbrev: 'n'
description: 'name of the publication.'
})
cmd_publisher.add_flag(Flag{
flag: .bool
required: false
name: 'edit'
description: 'will open vscode for collections & summary.'
})
cmd_publisher.add_flag(Flag{
flag: .bool
required: false
name: 'open'
abbrev: 'o'
description: 'will open the generated book.'
})
mut cmd_list := Command{
sort_flags: true
name: 'list_books'
execute: cmd_publisher_list_books
description: 'will list existing mdbooks'
pre_execute: pre_func
}
mut cmd_open := Command{
name: 'open'
execute: cmd_publisher_open
description: 'will open the publication with the provided name'
pre_execute: pre_func
}
cmd_open.add_flag(Flag{
flag: .string
name: 'name'
abbrev: 'n'
description: 'name of the publication.'
})
cmd_publisher.add_command(cmd_list)
cmd_publisher.add_command(cmd_open)
// cmdroot.add_command(cmd_publisher)
return cmd_publisher
}
fn cmd_publisher_list_books(cmd Command) ! {
console.print_header('Books:')
books := publisher.list_books()!
for book in books {
console.print_stdout(book.str())
}
}
fn cmd_publisher_open(cmd Command) ! {
name := cmd.flags.get_string('name') or { '' }
publisher.open(name)!
}
fn cmd_publisher_execute(cmd Command) ! {
mut name := cmd.flags.get_string('name') or { '' }
// mut url := cmd.flags.get_string('url') or { '' }
// mut path := cmd.flags.get_string('path') or { '' }
// if path.len > 0 || url.len > 0 {
// // execute the attached playbook
// mut plbook, _ := herocmds.plbook_run(cmd)!
// play(mut plbook)!
// // get name from the book.generate action
// // if name == '' {
// // mut a := plbook.action_get(actor: 'mdbook', name: 'define')!
// // name = a.params.get('name') or { '' }
// // }
// } else {
// publisher_help(cmd)
// }
if name == '' {
console.print_debug('did not find name of book to generate, check in heroscript or specify with --name')
publisher_help(cmd)
exit(1)
}
edit := cmd.flags.get_bool('edit') or { false }
open := cmd.flags.get_bool('open') or { false }
if edit || open {
// mdbook.book_open(name)!
}
if edit {
// publisher.book_edit(name)!
}
}
// fn pre_func(cmd Command) ! {
// herocmds.plbook_run(cmd)!
// }
fn publisher_help(cmd Command) {
console.clear()
console.print_header('Instructions for publisher:')
console.print_lf(1)
console.print_stdout(cmd.help_message())
console.print_lf(5)
}

117
lib/hero/publishing/play.v Normal file
View File

@@ -0,0 +1,117 @@
module publishing
import freeflowuniverse.herolib.core.playbook { Action }
import freeflowuniverse.herolib.data.paramsparser { Params }
import freeflowuniverse.herolib.develop.gittools
import freeflowuniverse.herolib.core.pathlib
import os
pub fn play(mut plbook playbook.PlayBook) ! {
// first lets configure are publisher
if mut action := plbook.action_get(actor: 'publisher', name: 'configure') {
play_configure(mut action)!
}
// lets add all the collections
for mut action in plbook.find(filter: 'publisher:new_collection')! {
mut p := action.params
play_new_collection(mut p)!
action.done = true
}
// then lets export the doctree with all its collections
publisher.export_tree()!
// now we can start defining books
for mut action in plbook.find(filter: 'book:define')! {
mut p := action.params
play_book_define(mut p)!
action.done = true
}
// finally lets publish defined books
for mut action in plbook.find(filter: 'book:publish')! {
p := action.params
spawn play_book_publish(p)
action.done = true
}
}
fn play_configure(mut action Action) ! {
mut p := action.params
// Variables removed as they were unused
if p.exists('buildroot') {
_ = p.get('buildroot')!
}
if p.exists('coderoot') {
_ = p.get('coderoot')!
}
if p.exists('publishroot') {
_ = p.get('publishroot')!
}
if p.exists('reset') {
_ = p.get_default_false('reset')
}
action.done = true
}
fn play_new_collection(mut p Params) ! {
url := p.get_default('url', '')!
path := p.get_default('path', '')!
// name removed as unused
reset := p.get_default_false('reset')
pull := p.get_default_false('pull')
mut tree := publisher.tree
tree.scan_concurrent(
path: path
git_url: url
git_reset: reset
git_pull: pull
)!
publisher.tree = tree
}
fn play_book_define(mut params Params) ! {
summary_url := params.get_default('summary_url', '')!
summary_path := if summary_url == '' {
params.get('summary_path') or {
return error('both summary url and summary path cannot be empty')
}
} else {
get_summary_path(summary_url)!
}
name := params.get('name')!
publisher.new_book(
name: name
title: params.get_default('title', name)!
collections: params.get_list('collections')!
summary_path: summary_path
)!
}
fn play_book_publish(p Params) ! {
name := p.get('name')!
params := p.decode[PublishParams]()!
// production removed as unused
publisher.publish(name, params)!
}
fn get_summary_path(summary_url string) !string {
mut gs := gittools.get()!
repo := gs.get_repo(url: summary_url, reset: false, pull: false)!
// get the path corresponding to the summary_url dir/file
summary_path := repo.get_path_of_url(summary_url)!
mut summary_dir := pathlib.get_dir(path: os.dir(summary_path))!
summary_file := summary_dir.file_get_ignorecase('summary.md') or {
summary_dir = summary_dir.parent()!
summary_dir.file_get_ignorecase('summary.md') or {
return error('summary from git needs to be dir or file: ${err}')
}
}
return summary_file.path
}

118
lib/hero/publishing/publisher.v Normal file
View File

@@ -0,0 +1,118 @@
module publishing
import os
import freeflowuniverse.herolib.core.pathlib { Path }
import freeflowuniverse.herolib.osal
import freeflowuniverse.herolib.data.doctree { Tree }
import freeflowuniverse.herolib.web.mdbook
__global (
publisher Publisher
)
pub struct Publisher {
pub mut:
tree Tree
books map[string]Book
root_path string = os.join_path(os.home_dir(), 'hero/publisher')
}
// returns the directory of a given collecation
fn (p Publisher) collection_directory(name string) ?Path {
mut cols_dir := p.collections_directory()
return cols_dir.dir_get(name) or { return none }
}
pub fn (p Publisher) collections_directory() Path {
collections_path := '${p.root_path}/collections'
return pathlib.get_dir(path: collections_path) or { panic('this should never happen ${err}') }
}
pub fn (p Publisher) build_directory() Path {
build_path := '${p.root_path}/build'
return pathlib.get_dir(path: build_path) or { panic('this should never happen ${err}') }
}
pub fn (p Publisher) publish_directory() Path {
publish_path := '${p.root_path}/publish'
return pathlib.get_dir(path: publish_path) or { panic('this should never happen ${err}') }
}
@[params]
pub struct PublishParams {
production bool
}
pub fn (p Publisher) publish(name string, params PublishParams) ! {
if name !in p.books {
return error('book ${name} doesnt exist')
}
p.books[name].publish(p.publish_directory().path, params)!
}
pub struct Book {
name string
title string
description string
path string
}
pub fn (book Book) publish(path string, params PublishParams) ! {
os.execute_opt('
cd ${book.path}
mdbook build --dest-dir ${path}/${book.name}')!
}
pub struct NewBook {
name string
title string
description string
summary_path string
collections []string
}
pub fn (p Publisher) new_book(book NewBook) ! {
mut mdbooks := mdbook.get()!
mut cfg := mdbooks
cfg.path_build = p.build_directory().path
cfg.path_publish = p.publish_directory().path
mut col_paths := []string{}
for col in book.collections {
col_dir := p.collection_directory(col) or {
return error('Collection ${col} not found in publisher tree')
}
col_paths << col_dir.path
}
_ := mdbooks.generate(
name: book.name
title: book.title
summary_path: book.summary_path
collections: col_paths
)!
publisher.books[book.name] = Book{
name: book.name
title: book.title
description: book.description
path: '${p.build_directory().path}/${book.name}'
}
}
pub fn (book Book) print() {
println('Book: ${book.name}\n- title: ${book.title}\n- description: ${book.description}\n- path: ${book.path}')
}
pub fn (p Publisher) open(name string) ! {
p.publish(name)!
cmd := 'open \'${p.publish_directory().path}/${name}/index.html\''
osal.exec(cmd: cmd)!
}
pub fn (p Publisher) export_tree() ! {
publisher.tree.export(destination: '${publisher.root_path}/collections')!
}
pub fn (p Publisher) list_books() ![]Book {
return p.books.values()
}

4
test_basic.vsh
View File

@@ -169,9 +169,6 @@ lib/code
lib/clients
lib/core
lib/develop
lib/markdownparser/
lib/ourdb/
lib/gittools
// lib/crypt
'
@@ -180,6 +177,7 @@ tests_ignore := '
notifier_test.v
clients/meilisearch
clients/zdb
clients/openai
systemd_process_test.v
'