Merge branch 'development' of https://github.com/freeflowuniverse/herolib into development

lib/clients/openai/.heroscript

@@ -1,7 +1,8 @@
+
 !!hero_code.generate_client
-    name: "openai"
+    name:'openai'
-    classname: "OpenAI"
+    classname:'OpenAI'
-    hasconfig: false
+    singleton:0
-    singleton: false
+    default:1
-    default: true
+	hasconfig:1
-    title: ""
+    reset:0

lib/clients/openai/openai_factory_.v

@@ -2,7 +2,6 @@ module openai
 
 import freeflowuniverse.herolib.core.base
 import freeflowuniverse.herolib.core.playbook
-import freeflowuniverse.herolib.ui.console
 
 __global (
 	openai_global  map[string]&OpenAI
@@ -14,35 +13,93 @@ __global (
 @[params]
 pub struct ArgsGet {
 pub mut:
-	name string
+	name string = 'default'
 }
 
 fn args_get(args_ ArgsGet) ArgsGet {
-	mut model := args_
+	mut args := args_
-	if model.name == '' {
+	if args.name == '' {
-		model.name = openai_default
+		args.name = openai_default
 	}
-	if model.name == '' {
+	if args.name == '' {
-		model.name = 'default'
+		args.name = 'default'
 	}
-	return model
+	return args
 }
 
 pub fn get(args_ ArgsGet) !&OpenAI {
 	mut args := args_get(args_)
 	if args.name !in openai_global {
-		if args.name == 'default' {
+		if !config_exists() {
-			if !config_exists(args) {
+			if default {
-				if default {
+				config_save()!
-					mut context := base.context() or { panic('bug') }
-					context.hero_config_set('openai', model.name, heroscript_default()!)!
-				}
 			}
-			load(args)!
 		}
+		config_load()!
 	}
 	return openai_global[args.name] or {
 		println(openai_global)
-		panic('could not get config for ${args.name} with name:${model.name}')
+		panic('bug in get from factory: ')
 	}
 }
+
+fn config_exists(args_ ArgsGet) bool {
+	mut args := args_get(args_)
+	mut context := base.context() or { panic('bug') }
+	return context.hero_config_exists('openai', args.name)
+}
+
+fn config_load(args_ ArgsGet) ! {
+	mut args := args_get(args_)
+	mut context := base.context()!
+	mut heroscript := context.hero_config_get('openai', args.name)!
+	play(heroscript: heroscript)!
+}
+
+fn config_save(args_ ArgsGet) ! {
+	mut args := args_get(args_)
+	mut context := base.context()!
+	context.hero_config_set('openai', args.name, heroscript_default()!)!
+}
+
+fn set(o OpenAI) ! {
+	mut o2 := obj_init(o)!
+	openai_global['default'] = &o2
+}
+
+@[params]
+pub struct PlayArgs {
+pub mut:
+	name       string = 'default'
+	heroscript string // if filled in then plbook will be made out of it
+	plbook     ?playbook.PlayBook
+	reset      bool
+
+	start     bool
+	stop      bool
+	restart   bool
+	delete    bool
+	configure bool // make sure there is at least one installed
+}
+
+pub fn play(args_ PlayArgs) ! {
+	mut args := args_
+
+	if args.heroscript == '' {
+		args.heroscript = heroscript_default()!
+	}
+	mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
+
+	mut install_actions := plbook.find(filter: 'openai.configure')!
+	if install_actions.len > 0 {
+		for install_action in install_actions {
+			mut p := install_action.params
+			cfg_play(p)!
+		}
+	}
+}
+
+// switch instance to be used for openai
+pub fn switch(name string) {
+	openai_default = name
+}

lib/clients/openai/openai_model.v

@@ -2,6 +2,7 @@ module openai
 
 import freeflowuniverse.herolib.data.paramsparser
 import freeflowuniverse.herolib.core.httpconnection
+import os
 
 pub const version = '1.14.3'
 const singleton = false
@@ -12,7 +13,7 @@ pub fn heroscript_default() !string {
 	heroscript := "
     !!openai.configure 
         name:'openai'
-        key: 'YOUR_API_KEY'
+        api_key: ${os.getenv('OPENAI_API_KEY')}
         "
 
 	return heroscript
@@ -49,6 +50,7 @@ pub fn (mut client OpenAI) connection() !&httpconnection.HTTPConnection {
 			name:  'openaiconnection_${client.name}'
 			url:   'https://api.openai.com/v1'
 			cache: false
+			retry: 20
 		)!
 		c2
 	}

lib/clients/rclone/.heroscript

@@ -1,7 +1,8 @@
+
 !!hero_code.generate_client
-    name: "rclone"
+    name:'rclone'
-    classname: "RCloneClient"
+    classname:'RCloneClient'
-    hasconfig: true
+    singleton:1
-    singleton: true
+    default:1
-    default: true
+	hasconfig:1
-    title: ""
+    reset:0

lib/clients/rclone/rclone_factory_.v

@@ -2,8 +2,6 @@ module rclone
 
 import freeflowuniverse.herolib.core.base
 import freeflowuniverse.herolib.core.playbook
-import freeflowuniverse.herolib.ui.console
-import freeflowuniverse.herolib.data.encoderhero
 
 __global (
 	rclone_global  map[string]&RCloneClient
@@ -12,59 +10,96 @@ __global (
 
 /////////FACTORY
 
-// set the model in mem and the config on the filesystem
+@[params]
-pub fn set(o RCloneClient) ! {
+pub struct ArgsGet {
-	mut o2 := obj_init(o)!
+pub mut:
-	rclone_global[o.name] = &o2
+	name string = 'default'
-	rclone_default = o.name
 }
 
-// check we find the config on the filesystem
+fn args_get(args_ ArgsGet) ArgsGet {
-pub fn exists(args_ ArgsGet) bool {
+	mut args := args_
-	mut model := args_get(args_)
+	if args.name == '' {
+		args.name = rclone_default
+	}
+	if args.name == '' {
+		args.name = 'default'
+	}
+	return args
+}
+
+pub fn get(args_ ArgsGet) !&RCloneClient {
+	mut args := args_get(args_)
+	if args.name !in rclone_global {
+		if !config_exists() {
+			if default {
+				config_save()!
+			}
+		}
+		config_load()!
+	}
+	return rclone_global[args.name] or {
+		println(rclone_global)
+		panic('bug in get from factory: ')
+	}
+}
+
+fn config_exists(args_ ArgsGet) bool {
+	mut args := args_get(args_)
 	mut context := base.context() or { panic('bug') }
-	return context.hero_config_exists('rclone', model.name)
+	return context.hero_config_exists('rclone', args.name)
 }
 
-// load the config error if it doesn't exist
+fn config_load(args_ ArgsGet) ! {
-pub fn load(args_ ArgsGet) ! {
+	mut args := args_get(args_)
-	mut model := args_get(args_)
 	mut context := base.context()!
-	mut heroscript := context.hero_config_get('rclone', model.name)!
+	mut heroscript := context.hero_config_get('rclone', args.name)!
 	play(heroscript: heroscript)!
 }
 
-// save the config to the filesystem in the context
+fn config_save(args_ ArgsGet) ! {
-pub fn save(o RCloneClient) ! {
+	mut args := args_get(args_)
 	mut context := base.context()!
-	heroscript := encoderhero.encode[RCloneClient](o)!
+	context.hero_config_set('rclone', args.name, heroscript_default()!)!
-	context.hero_config_set('rclone', model.name, heroscript)!
+}
+
+fn set(o RCloneClient) ! {
+	mut o2 := obj_init(o)!
+	rclone_global['default'] = &o2
 }
 
 @[params]
 pub struct PlayArgs {
 pub mut:
+	name       string = 'default'
 	heroscript string // if filled in then plbook will be made out of it
 	plbook     ?playbook.PlayBook
 	reset      bool
+
+	start     bool
+	stop      bool
+	restart   bool
+	delete    bool
+	configure bool // make sure there is at least one installed
 }
 
 pub fn play(args_ PlayArgs) ! {
-	mut model := args_
+	mut args := args_
 
-	if model.heroscript == '' {
+	if args.heroscript == '' {
-		model.heroscript = heroscript_default()!
+		args.heroscript = heroscript_default()!
 	}
-	mut plbook := model.plbook or { playbook.new(text: model.heroscript)! }
+	mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
 
-	mut configure_actions := plbook.find(filter: 'rclone.configure')!
+	mut install_actions := plbook.find(filter: 'rclone.configure')!
-	if configure_actions.len > 0 {
+	if install_actions.len > 0 {
-		for config_action in configure_actions {
+		for install_action in install_actions {
-			mut p := config_action.params
+			mut p := install_action.params
-			mycfg := cfg_play(p)!
+			cfg_play(p)!
-			console.print_debug('install action rclone.configure\n${mycfg}')
-			set(mycfg)!
-			save(mycfg)!
 		}
 	}
 }
+
+// switch instance to be used for rclone
+pub fn switch(name string) {
+	rclone_default = name
+}

lib/core/base/context.v

@@ -70,8 +70,8 @@ pub fn (mut self Context) redis() !&redisclient.Redis {
 			// make sure we are on the right db
 			r.selectdb(int(self.config.id))!
 		}
-		self.redis_ = &r
+		self.redis_ = r
-		&r
+		r
 	}
 
 	return r2

lib/core/logger/log_test.v

@@ -81,16 +81,25 @@ fn test_logger() {
 		create: false
 	)!
 
-	println('/tmp/testlogs/${files[0]}')
-
 	content := file.read()!.trim_space()
 
-	items := logger.search()!
+	items_stdout := logger.search(
-	assert items.len == 6 // still wrong: TODO
+		timestamp_from: ourtime.new('2022-11-1 20:14:35')!
+		timestamp_to:   ourtime.new('2025-11-1 20:14:35')!
+		logtype:        .stdout
+	)!
+	assert items_stdout.len == 2
+
+	items_error := logger.search(
+		timestamp_from: ourtime.new('2022-11-1 20:14:35')!
+		timestamp_to:   ourtime.new('2025-11-1 20:14:35')!
+		logtype:        .error
+	)!
+	assert items_error.len == 4
 }
 
 fn testsuite_end() {
-	if os.exists('/tmp/testlogs') {
+	// if os.exists('/tmp/testlogs') {
-		os.rmdir_all('/tmp/testlogs')!
+	// 	os.rmdir_all('/tmp/testlogs')!
-	}
+	// }
 }

lib/core/logger/search.v

@@ -30,7 +30,6 @@ pub fn (mut l Logger) search(args_ SearchArgs) ![]LogItem {
 	// Get time range
 	from_time := timestamp_from.unix()
 	to_time := timestamp_to.unix()
-
 	if from_time > to_time {
 		return error('from_time cannot be after to_time: ${from_time} <  ${to_time}')
 	}
@@ -82,20 +81,30 @@ pub fn (mut l Logger) search(args_ SearchArgs) ![]LogItem {
 				continue
 			}
 
+			if collecting && line.len > 14 && line[13] == `-` {
+				process(mut result, current_item, current_time, args, from_time, to_time)!
+				collecting = false
+			}
+
 			// Parse log line
 			is_error := line.starts_with('E')
 			if !collecting {
 				// Start new item
 				current_item = LogItem{
 					timestamp: current_time
-					cat:       line_trim[2..12].trim_space()
+					cat:       line[2..12].trim_space()
-					log:       line_trim[15..].trim_space()
+					log:       line[15..].trim_space()
 					logtype:   if is_error { .error } else { .stdout }
 				}
+				// println('new current item: ${current_item}')
 				collecting = true
 			} else {
 				// Continuation line
-				current_item.log += '\n' + line_trim[15..]
+				if line_trim.len < 16 {
+					current_item.log += '\n'
+				} else {
+					current_item.log += '\n' + line[15..]
+				}
 			}
 		}
 

lib/core/playbook/factory.v

@@ -29,7 +29,7 @@ pub mut:
 pub fn new(args_ PlayBookNewArgs) !PlayBook {
 	mut args := args_
 
-	mut c := base.context()!
+	mut c := base.context() or { return error('failed to get context: ${err}') }
 
 	mut s := c.session_new()!
 

lib/core/playcmds/play_dagu_test.v

@@ -27,5 +27,5 @@ const dagu_script = "
 fn test_play_dagu() ! {
 	mut plbook := playbook.new(text: dagu_script)!
 	play_dagu(mut plbook)!
-	panic('s')
+	// panic('s')
 }

lib/core/playcmds/play_mdbook.v

@@ -1,135 +0,0 @@
-module playcmds
-
-import freeflowuniverse.herolib.web.mdbook
-import freeflowuniverse.herolib.data.doctree
-import freeflowuniverse.herolib.core.playbook
-import os
-
-pub fn play_mdbook(mut plbook playbook.PlayBook) ! {
-	mut buildroot := '${os.home_dir()}/hero/var/mdbuild'
-	mut publishroot := '${os.home_dir()}/hero/www/info'
-	mut coderoot := ''
-	// mut install := false
-	mut reset := false
-	mut pull := false
-
-	// check if any actions for doctree, if not then nothing to do here
-	// dtactions := plbook.find(filter: 'doctree.')!
-	// if dtactions.len == 0 {
-	// 	console.print_debug("can't find doctree.add statements, nothing to do")
-	// 	return
-	// }
-
-	mut config_actions := plbook.find(filter: 'books:configure')!
-
-	if config_actions.len > 1 {
-		return error('can only have 1 config action for books')
-	} else if config_actions.len == 1 {
-		mut p := config_actions[0].params
-		if p.exists('buildroot') {
-			buildroot = p.get('buildroot')!
-		}
-		if p.exists('coderoot') {
-			coderoot = p.get('coderoot')!
-		}
-		if p.exists('publishroot') {
-			publishroot = p.get('publishroot')!
-		}
-		if p.exists('reset') {
-			reset = p.get_default_false('reset')
-		}
-		config_actions[0].done = true
-	}
-
-	mut trees := map[string]&doctree.Tree{}
-	for mut action in plbook.find(filter: 'doctree:new')! {
-		mut p := action.params
-		name := p.get('name')!
-		fail_on_error := p.get_default_false('fail_on_error')
-		if name in trees {
-			return error('tree with name ${name} already exists')
-		}
-
-		tree := doctree.new(name: name, fail_on_error: fail_on_error)!
-		trees[name] = tree
-	}
-
-	for mut action in plbook.find(filter: 'doctree:add')! {
-		mut p := action.params
-		url := p.get_default('url', '')!
-		path := p.get_default('path', '')!
-		name := p.get_default('name', '')!
-
-		if trees.len == 0 {
-			return error('no tree found')
-		}
-
-		mut tree := if name != '' {
-			trees[name] or { return error('tree ${name} not found') }
-		} else {
-			trees.values()[0]
-		}
-
-		tree.scan(
-			path:      path
-			git_url:   url
-			git_reset: reset
-			git_root:  coderoot
-			git_pull:  pull
-		)!
-		action.done = true
-	}
-
-	for mut action in plbook.find(filter: 'doctree:export')! {
-		mut p := action.params
-		build_path := p.get('path')!
-		toreplace := p.get_default('replace', '')!
-		reset2 := p.get_default_false('reset')
-		name := p.get('name')!
-		mut tree := trees[name] or { return error('tree: ${name} not found') }
-
-		tree.export(
-			destination: build_path
-			reset:       reset2
-			toreplace:   toreplace
-		)!
-		action.done = true
-	}
-
-	for mut action in plbook.find(filter: 'mdbook:export')! {
-		mut p := action.params
-		name := p.get('name')!
-		summary_url := p.get_default('summary_url', '')!
-		summary_path := p.get_default('summary_path', '')!
-		title := p.get_default('title', name)!
-		publish_path := p.get_default('publish_path', '${publishroot}/${name}')!
-		build_path := p.get_default('build_path', '${buildroot}/${name}')!
-		printbook := p.get_default_false('printbook')
-		foldlevel := p.get_int_default('foldlevel', 0)!
-		production := p.get_default_false('production')
-		reset3 := p.get_default_true('reset')
-		collections := p.get_list_default('collections', [])!
-
-		if summary_url == '' && summary_path == '' {
-			return error('Both summary url and path cannot be empty at the same time')
-		}
-
-		mut mdbooks := mdbook.get()!
-
-		mdbooks.path_build = buildroot
-		mdbooks.path_publish = publishroot
-
-		mdbooks.generate(
-			name:         name
-			title:        title
-			summary_path: summary_path
-			publish_path: publish_path
-			build_path:   build_path
-			printbook:    printbook
-			foldlevel:    foldlevel
-			production:   production
-			collections:  collections
-		)!
-		action.done = true
-	}
-}

lib/core/playcmds/play_mdbook_test.v

@@ -1,85 +0,0 @@
-module playcmds
-
-import freeflowuniverse.herolib.core.playbook
-import freeflowuniverse.herolib.core.playcmds
-import freeflowuniverse.herolib.core.pathlib
-
-fn test_play_mdbook() {
-	mut summary_path := pathlib.get_file(path: '/tmp/mdbook_test/SUMMARY.md', create: true)!
-	summar_content := '
-- [Page number 1](fruits/apple.md)
-- [fruit intro](fruits/intro.md)
-- [rpc page](rpc/tfchain.md)
-- [vegies](test_vegetables/tomato.md)
-'
-	summary_path.write(summar_content)!
-
-	mut p := pathlib.get_file(path: '/tmp/heroscript/do.hero', create: true)!
-	// 	script := "
-	// !!doctree.new
-	// 	name: 'tree1'
-
-	// !!doctree.add
-	// 	name: 'tree1'
-	// 	url:'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/actions'
-
-	// !!doctree.add
-	// 	name: 'tree1'
-	// 	url: 'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/fruits'
-
-	// !!doctree.add
-	// 	name: 'tree1'
-	// 	url: 'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/rpc'
-
-	// !!doctree.export
-	// 	name: 'tree1'
-	// 	path: '/tmp/export_tree1'
-
-	// !!doctree.new
-	// 	name: 'tree2'
-	// 	fail_on_error: true
-
-	// !!doctree.add
-	// 	name: 'tree2'
-	// 	url: 'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/vegetables'
-
-	// !!doctree.export
-	// 	name: 'tree2'
-	// 	path: '/tmp/export_tree2'
-
-	// !!mdbook.export
-	//     title:'ThreeFold Technology'
-	//     name:'tech'
-	//     summary_path:'${summary_path.path}'
-	//     collections:'/tmp/export_tree1,/tmp/export_tree2'
-	//     dest: '/tmp/mdbook_export'
-	//     production:0 //means we put it in summary
-	// "
-
-	s2 := "
-!!doctree.new
-    name: 'info_tfgrid'
-    fail_on_error: false
-
-!!doctree.add 
-    name:'info_tfgrid' 
-    url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/main/collections'
-
-
-!!doctree.export 
-    name:'info_tfgrid' 
-    path:'~/hero/var/collections/info_tfgrid' 
-
-
-!!mdbook.export
-    title:'ThreeFold Technology'
-    name:'tech'
-    summary_url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/development/books/tech/SUMMARY.md' 
-    collections:'~/hero/var/collections/info_tfgrid' 
-    production:0 //means we put it in summary
-"
-	p.write(s2)!
-
-	mut plbook := playbook.new(path: '/tmp/heroscript')!
-	playcmds.play_mdbook(mut plbook)!
-}

lib/core/playcmds/play_publisher.v

@@ -0,0 +1,8 @@
+module playcmds
+
+import freeflowuniverse.herolib.core.playbook
+import freeflowuniverse.herolib.hero.publishing
+
+pub fn play_publisher(mut plbook playbook.PlayBook) ! {
+	publishing.play(mut plbook)!
+}

lib/core/playcmds/play_publisher_test.v

@@ -0,0 +1,34 @@
+module playcmds
+
+import freeflowuniverse.herolib.core.playbook
+import freeflowuniverse.herolib.core.playcmds
+import freeflowuniverse.herolib.core.pathlib
+import os
+
+fn test_play_publisher() {
+	mut p := pathlib.get_file(path: '/tmp/heroscript/do.hero', create: true)!
+
+	s2 := "
+
+!!publisher.new_collection
+	url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/main/collections'
+	reset: false
+	pull: true
+
+
+!!book.define 
+    name:'info_tfgrid' 
+    summary_url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/development/books/tech/SUMMARY.md' 
+	title:'ThreeFold Technology'
+	collections: 'about,dashboard,farmers,library,partners_utilization,tech,p2p'
+
+
+!!book.publish
+    name:'tech'
+	production: false
+"
+	p.write(s2)!
+
+	mut plbook := playbook.new(path: '/tmp/heroscript')!
+	playcmds.play_publisher(mut plbook)!
+}

lib/core/redisclient/factory.v

@@ -3,11 +3,11 @@ module redisclient
 // original code see https://github.com/patrickpissurno/vredis/blob/master/vredis_test.v
 // credits see there as well (-:
 import net
-// import sync
+import sync
 // import strconv
 
 __global (
-	redis_connections []Redis
+	redis_connections []&Redis
 )
 
 const default_read_timeout = net.infinite_timeout
@@ -18,15 +18,16 @@ pub:
 	addr string
 mut:
 	socket net.TcpConn
+	mtx    sync.RwMutex
 }
 
 // https://redis.io/topics/protocol
 // examples:
 //   localhost:6379
 //   /tmp/redis-default.sock
-pub fn new(addr string) !Redis {
+pub fn new(addr string) !&Redis {
-	// lock redis_connections {	
+	// lock redis_cowritennections {	
-	for mut conn in redis_connections {
+	for conn in redis_connections {
 		if conn.addr == addr {
 			return conn
 		}
@@ -34,10 +35,13 @@ pub fn new(addr string) !Redis {
 	// means there is no connection yet
 	mut r := Redis{
 		addr: addr
+		mtx:  sync.RwMutex{}
 	}
+	r.mtx.init()
+
 	r.socket_connect()!
-	redis_connections << r
+	redis_connections << &r
-	return r
+	return &r
 	//}
 	// panic("bug")
 }
@@ -47,7 +51,7 @@ pub fn reset() ! {
 	for mut conn in redis_connections {
 		conn.disconnect()
 	}
-	redis_connections = []Redis{}
+	redis_connections = []&Redis{}
 	//}
 }
 

lib/core/redisclient/redisclient_core.v

@@ -18,7 +18,7 @@ pub fn get_redis_url(url string) !RedisURL {
 	}
 }
 
-pub fn core_get(url RedisURL) !Redis {
+pub fn core_get(url RedisURL) !&Redis {
 	mut r := new('${url.address}:${url.port}')!
 	return r
 }

lib/core/redisclient/redisclient_encode.v

@@ -2,7 +2,7 @@ module redisclient
 
 import freeflowuniverse.herolib.data.resp
 
-pub fn (mut r Redis) get_response() !resp.RValue {
+fn (mut r Redis) get_response() !resp.RValue {
 	line := r.read_line()!
 
 	if line.starts_with('-') {
@@ -63,7 +63,7 @@ pub fn (mut r Redis) get_response() !resp.RValue {
 
 // TODO: needs to use the resp library
 
-pub fn (mut r Redis) get_int() !int {
+fn (mut r Redis) get_int() !int {
 	line := r.read_line()!
 	if line.starts_with(':') {
 		return line[1..].int()
@@ -72,7 +72,7 @@ pub fn (mut r Redis) get_int() !int {
 	}
 }
 
-pub fn (mut r Redis) get_list_int() ![]int {
+fn (mut r Redis) get_list_int() ![]int {
 	line := r.read_line()!
 	mut res := []int{}
 
@@ -89,7 +89,7 @@ pub fn (mut r Redis) get_list_int() ![]int {
 	}
 }
 
-pub fn (mut r Redis) get_list_str() ![]string {
+fn (mut r Redis) get_list_str() ![]string {
 	line := r.read_line()!
 	mut res := []string{}
 
@@ -106,7 +106,7 @@ pub fn (mut r Redis) get_list_str() ![]string {
 	}
 }
 
-pub fn (mut r Redis) get_string() !string {
+fn (mut r Redis) get_string() !string {
 	line := r.read_line()!
 	if line.starts_with('+') {
 		// console.print_debug("getstring:'${line[1..]}'")
@@ -120,12 +120,12 @@ pub fn (mut r Redis) get_string() !string {
 	}
 }
 
-pub fn (mut r Redis) get_string_nil() !string {
+fn (mut r Redis) get_string_nil() !string {
 	r2 := r.get_bytes_nil()!
 	return r2.bytestr()
 }
 
-pub fn (mut r Redis) get_bytes_nil() ![]u8 {
+fn (mut r Redis) get_bytes_nil() ![]u8 {
 	line := r.read_line()!
 	if line.starts_with('+') {
 		return line[1..].bytes()
@@ -140,12 +140,12 @@ pub fn (mut r Redis) get_bytes_nil() ![]u8 {
 	}
 }
 
-pub fn (mut r Redis) get_bool() !bool {
+fn (mut r Redis) get_bool() !bool {
 	i := r.get_int()!
 	return i == 1
 }
 
-pub fn (mut r Redis) get_bytes() ![]u8 {
+fn (mut r Redis) get_bytes() ![]u8 {
 	line := r.read_line()!
 	if line.starts_with('$') {
 		return r.get_bytes_from_line(line)

lib/core/redisclient/redisclient_internal.v

@@ -53,7 +53,7 @@ fn (mut r Redis) socket_check() ! {
 	}
 }
 
-pub fn (mut r Redis) read_line() !string {
+fn (mut r Redis) read_line() !string {
 	return r.socket.read_line().trim_right('\r\n')
 }
 
@@ -99,7 +99,7 @@ fn (mut r Redis) write_line(data []u8) ! {
 }
 
 // write resp value to the redis channel
-pub fn (mut r Redis) write_rval(val resp.RValue) ! {
+fn (mut r Redis) write_rval(val resp.RValue) ! {
 	r.write(val.encode())!
 }
 

lib/core/redisclient/redisclient_sadd_test.v

@@ -3,7 +3,7 @@ import freeflowuniverse.herolib.core.redisclient
 fn setup() !&redisclient.Redis {
 	mut redis := redisclient.core_get()!
 	redis.selectdb(10) or { panic(err) }
-	return &redis
+	return redis
 }
 
 fn cleanup(mut redis redisclient.Redis) ! {

lib/core/redisclient/redisclient_send.v

@@ -5,6 +5,10 @@ import freeflowuniverse.herolib.ui.console
 
 // send list of strings, expect OK back
 pub fn (mut r Redis) send_expect_ok(items []string) ! {
+	r.mtx.lock()
+	defer {
+		r.mtx.unlock()
+	}
 	r.write_cmds(items)!
 	res := r.get_string()!
 	if res != 'OK' {
@@ -15,23 +19,39 @@ pub fn (mut r Redis) send_expect_ok(items []string) ! {
 
 // send list of strings, expect int back
 pub fn (mut r Redis) send_expect_int(items []string) !int {
+	r.mtx.lock()
+	defer {
+		r.mtx.unlock()
+	}
 	r.write_cmds(items)!
 	return r.get_int()
 }
 
 pub fn (mut r Redis) send_expect_bool(items []string) !bool {
+	r.mtx.lock()
+	defer {
+		r.mtx.unlock()
+	}
 	r.write_cmds(items)!
 	return r.get_bool()
 }
 
 // send list of strings, expect string back
 pub fn (mut r Redis) send_expect_str(items []string) !string {
+	r.mtx.lock()
+	defer {
+		r.mtx.unlock()
+	}
 	r.write_cmds(items)!
 	return r.get_string()
 }
 
 // send list of strings, expect string or nil back
 pub fn (mut r Redis) send_expect_strnil(items []string) !string {
+	r.mtx.lock()
+	defer {
+		r.mtx.unlock()
+	}
 	r.write_cmds(items)!
 	d := r.get_string_nil()!
 	return d
@@ -39,16 +59,28 @@ pub fn (mut r Redis) send_expect_strnil(items []string) !string {
 
 // send list of strings, expect list of strings back
 pub fn (mut r Redis) send_expect_list_str(items []string) ![]string {
+	r.mtx.lock()
+	defer {
+		r.mtx.unlock()
+	}
 	r.write_cmds(items)!
 	return r.get_list_str()
 }
 
 pub fn (mut r Redis) send_expect_list_int(items []string) ![]int {
+	r.mtx.lock()
+	defer {
+		r.mtx.unlock()
+	}
 	r.write_cmds(items)!
 	return r.get_list_int()
 }
 
 pub fn (mut r Redis) send_expect_list(items []string) ![]resp.RValue {
+	r.mtx.lock()
+	defer {
+		r.mtx.unlock()
+	}
 	r.write_cmds(items)!
 	res := r.get_response()!
 	return resp.get_redis_array(res)

lib/core/redisclient/redisclient_test.v

@@ -8,7 +8,7 @@ fn setup() !&redisclient.Redis {
 	mut redis := redisclient.core_get()!
 	// Select db 10 to be away from default one '0'
 	redis.selectdb(10) or { panic(err) }
-	return &redis
+	return redis
 }
 
 fn cleanup(mut redis redisclient.Redis) ! {

lib/develop/gittools/repository_cache.v

@@ -3,7 +3,7 @@ module gittools
 import json
 import freeflowuniverse.herolib.core.redisclient
 
-fn redis_get() redisclient.Redis {
+fn redis_get() &redisclient.Redis {
 	mut redis_client := redisclient.core_get() or { panic(err) }
 	return redis_client
 }

lib/hero/publishing/command.v

@@ -0,0 +1,141 @@
+module publishing
+
+import cli { Command, Flag }
+import freeflowuniverse.herolib.ui.console
+
+// path string //if location on filessytem, if exists, this has prio on git_url
+// git_url   string // location of where the hero scripts are
+// git_pull     bool // means when getting new repo will pull even when repo is already there
+// git_pullreset bool // means we will force a pull and reset old content
+// coderoot string //the location of coderoot if its another one
+pub fn cmd_publisher(pre_func fn (Command) !) Command {
+	mut cmd_publisher := Command{
+		name:          'publisher'
+		usage:         '
+## Manage your publications
+
+example:
+
+hero publisher -u https://git.ourworld.tf/ourworld_holding/info_ourworld/src/branch/develop/heroscript
+
+If you do -gp it will pull newest book content from git and give error if there are local changes.
+If you do -gr it will pull newest book content from git and overwrite local changes (careful).
+
+		'
+		description:   'create, edit, show mdbooks'
+		required_args: 0
+		execute:       cmd_publisher_execute
+		pre_execute:   pre_func
+	}
+
+	// cmd_run_add_flags(mut cmd_publisher)
+
+	cmd_publisher.add_flag(Flag{
+		flag:        .string
+		name:        'name'
+		abbrev:      'n'
+		description: 'name of the publication.'
+	})
+
+	cmd_publisher.add_flag(Flag{
+		flag:        .bool
+		required:    false
+		name:        'edit'
+		description: 'will open vscode for collections & summary.'
+	})
+
+	cmd_publisher.add_flag(Flag{
+		flag:        .bool
+		required:    false
+		name:        'open'
+		abbrev:      'o'
+		description: 'will open the generated book.'
+	})
+
+	mut cmd_list := Command{
+		sort_flags:  true
+		name:        'list_books'
+		execute:     cmd_publisher_list_books
+		description: 'will list existing mdbooks'
+		pre_execute: pre_func
+	}
+
+	mut cmd_open := Command{
+		name:        'open'
+		execute:     cmd_publisher_open
+		description: 'will open the publication with the provided name'
+		pre_execute: pre_func
+	}
+
+	cmd_open.add_flag(Flag{
+		flag:        .string
+		name:        'name'
+		abbrev:      'n'
+		description: 'name of the publication.'
+	})
+
+	cmd_publisher.add_command(cmd_list)
+	cmd_publisher.add_command(cmd_open)
+	// cmdroot.add_command(cmd_publisher)
+	return cmd_publisher
+}
+
+fn cmd_publisher_list_books(cmd Command) ! {
+	console.print_header('Books:')
+	books := publisher.list_books()!
+	for book in books {
+		console.print_stdout(book.str())
+	}
+}
+
+fn cmd_publisher_open(cmd Command) ! {
+	name := cmd.flags.get_string('name') or { '' }
+	publisher.open(name)!
+}
+
+fn cmd_publisher_execute(cmd Command) ! {
+	mut name := cmd.flags.get_string('name') or { '' }
+
+	// mut url := cmd.flags.get_string('url') or { '' }
+	// mut path := cmd.flags.get_string('path') or { '' }
+	// if path.len > 0 || url.len > 0 {
+	// 	// execute the attached playbook
+	// 	mut plbook, _ := herocmds.plbook_run(cmd)!
+	// 	play(mut plbook)!
+	// 	// get name from the book.generate action
+	// 	// if name == '' {
+	// 	// 	mut a := plbook.action_get(actor: 'mdbook', name: 'define')!
+	// 	// 	name = a.params.get('name') or { '' }
+	// 	// }
+	// } else {
+	// 	publisher_help(cmd)
+	// }
+
+	if name == '' {
+		console.print_debug('did not find name of book to generate, check in heroscript or specify with --name')
+		publisher_help(cmd)
+		exit(1)
+	}
+
+	edit := cmd.flags.get_bool('edit') or { false }
+	open := cmd.flags.get_bool('open') or { false }
+	if edit || open {
+		// mdbook.book_open(name)!
+	}
+
+	if edit {
+		// publisher.book_edit(name)!
+	}
+}
+
+// fn pre_func(cmd Command) ! {
+// 	herocmds.plbook_run(cmd)!
+// }
+
+fn publisher_help(cmd Command) {
+	console.clear()
+	console.print_header('Instructions for publisher:')
+	console.print_lf(1)
+	console.print_stdout(cmd.help_message())
+	console.print_lf(5)
+}

lib/hero/publishing/play.v

@@ -0,0 +1,117 @@
+module publishing
+
+import freeflowuniverse.herolib.core.playbook { Action }
+import freeflowuniverse.herolib.data.paramsparser { Params }
+import freeflowuniverse.herolib.develop.gittools
+import freeflowuniverse.herolib.core.pathlib
+import os
+
+pub fn play(mut plbook playbook.PlayBook) ! {
+	// first lets configure are publisher
+	if mut action := plbook.action_get(actor: 'publisher', name: 'configure') {
+		play_configure(mut action)!
+	}
+
+	// lets add all the collections
+	for mut action in plbook.find(filter: 'publisher:new_collection')! {
+		mut p := action.params
+		play_new_collection(mut p)!
+		action.done = true
+	}
+
+	// then lets export the doctree with all its collections
+	publisher.export_tree()!
+
+	// now we can start defining books
+	for mut action in plbook.find(filter: 'book:define')! {
+		mut p := action.params
+		play_book_define(mut p)!
+		action.done = true
+	}
+
+	// finally lets publish defined books
+	for mut action in plbook.find(filter: 'book:publish')! {
+		p := action.params
+		spawn play_book_publish(p)
+		action.done = true
+	}
+}
+
+fn play_configure(mut action Action) ! {
+	mut p := action.params
+	// Variables removed as they were unused
+	if p.exists('buildroot') {
+		_ = p.get('buildroot')!
+	}
+	if p.exists('coderoot') {
+		_ = p.get('coderoot')!
+	}
+	if p.exists('publishroot') {
+		_ = p.get('publishroot')!
+	}
+	if p.exists('reset') {
+		_ = p.get_default_false('reset')
+	}
+	action.done = true
+}
+
+fn play_new_collection(mut p Params) ! {
+	url := p.get_default('url', '')!
+	path := p.get_default('path', '')!
+	// name removed as unused
+	reset := p.get_default_false('reset')
+	pull := p.get_default_false('pull')
+
+	mut tree := publisher.tree
+	tree.scan_concurrent(
+		path:      path
+		git_url:   url
+		git_reset: reset
+		git_pull:  pull
+	)!
+	publisher.tree = tree
+}
+
+fn play_book_define(mut params Params) ! {
+	summary_url := params.get_default('summary_url', '')!
+	summary_path := if summary_url == '' {
+		params.get('summary_path') or {
+			return error('both summary url and summary path cannot be empty')
+		}
+	} else {
+		get_summary_path(summary_url)!
+	}
+
+	name := params.get('name')!
+	publisher.new_book(
+		name:         name
+		title:        params.get_default('title', name)!
+		collections:  params.get_list('collections')!
+		summary_path: summary_path
+	)!
+}
+
+fn play_book_publish(p Params) ! {
+	name := p.get('name')!
+	params := p.decode[PublishParams]()!
+	// production removed as unused
+	publisher.publish(name, params)!
+}
+
+fn get_summary_path(summary_url string) !string {
+	mut gs := gittools.get()!
+	repo := gs.get_repo(url: summary_url, reset: false, pull: false)!
+
+	// get the path corresponding to the summary_url dir/file
+	summary_path := repo.get_path_of_url(summary_url)!
+	mut summary_dir := pathlib.get_dir(path: os.dir(summary_path))!
+
+	summary_file := summary_dir.file_get_ignorecase('summary.md') or {
+		summary_dir = summary_dir.parent()!
+		summary_dir.file_get_ignorecase('summary.md') or {
+			return error('summary from git needs to be dir or file: ${err}')
+		}
+	}
+
+	return summary_file.path
+}

lib/hero/publishing/publisher.v

@@ -0,0 +1,118 @@
+module publishing
+
+import os
+import freeflowuniverse.herolib.core.pathlib { Path }
+import freeflowuniverse.herolib.osal
+import freeflowuniverse.herolib.data.doctree { Tree }
+import freeflowuniverse.herolib.web.mdbook
+
+__global (
+	publisher Publisher
+)
+
+pub struct Publisher {
+pub mut:
+	tree      Tree
+	books     map[string]Book
+	root_path string = os.join_path(os.home_dir(), 'hero/publisher')
+}
+
+// returns the directory of a given collecation
+fn (p Publisher) collection_directory(name string) ?Path {
+	mut cols_dir := p.collections_directory()
+	return cols_dir.dir_get(name) or { return none }
+}
+
+pub fn (p Publisher) collections_directory() Path {
+	collections_path := '${p.root_path}/collections'
+	return pathlib.get_dir(path: collections_path) or { panic('this should never happen ${err}') }
+}
+
+pub fn (p Publisher) build_directory() Path {
+	build_path := '${p.root_path}/build'
+	return pathlib.get_dir(path: build_path) or { panic('this should never happen ${err}') }
+}
+
+pub fn (p Publisher) publish_directory() Path {
+	publish_path := '${p.root_path}/publish'
+	return pathlib.get_dir(path: publish_path) or { panic('this should never happen ${err}') }
+}
+
+@[params]
+pub struct PublishParams {
+	production bool
+}
+
+pub fn (p Publisher) publish(name string, params PublishParams) ! {
+	if name !in p.books {
+		return error('book ${name} doesnt exist')
+	}
+	p.books[name].publish(p.publish_directory().path, params)!
+}
+
+pub struct Book {
+	name        string
+	title       string
+	description string
+	path        string
+}
+
+pub fn (book Book) publish(path string, params PublishParams) ! {
+	os.execute_opt('	
+		cd ${book.path}
+		mdbook build --dest-dir ${path}/${book.name}')!
+}
+
+pub struct NewBook {
+	name         string
+	title        string
+	description  string
+	summary_path string
+	collections  []string
+}
+
+pub fn (p Publisher) new_book(book NewBook) ! {
+	mut mdbooks := mdbook.get()!
+	mut cfg := mdbooks
+	cfg.path_build = p.build_directory().path
+	cfg.path_publish = p.publish_directory().path
+
+	mut col_paths := []string{}
+	for col in book.collections {
+		col_dir := p.collection_directory(col) or {
+			return error('Collection ${col} not found in publisher tree')
+		}
+		col_paths << col_dir.path
+	}
+
+	_ := mdbooks.generate(
+		name:         book.name
+		title:        book.title
+		summary_path: book.summary_path
+		collections:  col_paths
+	)!
+	publisher.books[book.name] = Book{
+		name:        book.name
+		title:       book.title
+		description: book.description
+		path:        '${p.build_directory().path}/${book.name}'
+	}
+}
+
+pub fn (book Book) print() {
+	println('Book: ${book.name}\n- title: ${book.title}\n- description: ${book.description}\n- path: ${book.path}')
+}
+
+pub fn (p Publisher) open(name string) ! {
+	p.publish(name)!
+	cmd := 'open \'${p.publish_directory().path}/${name}/index.html\''
+	osal.exec(cmd: cmd)!
+}
+
+pub fn (p Publisher) export_tree() ! {
+	publisher.tree.export(destination: '${publisher.root_path}/collections')!
+}
+
+pub fn (p Publisher) list_books() ![]Book {
+	return p.books.values()
+}

test_basic.vsh

@@ -169,9 +169,6 @@ lib/code
 lib/clients
 lib/core
 lib/develop
-lib/markdownparser/
-lib/ourdb/
-lib/gittools
 // lib/crypt
 '
 
@@ -180,6 +177,7 @@ tests_ignore := '
 notifier_test.v
 clients/meilisearch
 clients/zdb
+clients/openai
 systemd_process_test.v
 '