the base

manual/best_practices/osal/silence.md

@@ -0,0 +1,21 @@
+# silence
+
+the following code shows how we can surpress all output, errors should still go to stderr (to be tested)
+
+the example is a .vsh script note the arguments to v, this also makes sure there are no notices shown.
+
+```go
+#!/usr/bin/env -S v -n -w -enable-globals run
+
+import freeflowuniverse.herolib.osal
+import freeflowuniverse.herolib.ui.console
+
+console.silent_set()
+mut job2 := osal.exec(cmd: 'ls /',debug:true)!
+println("I got nothing above")
+
+console.silent_unset()
+println("now I will get output")
+
+osal.exec(cmd: 'ls /',debug:true)!
+```

manual/best_practices/scripts/scripts.md

@@ -0,0 +1,61 @@
+# Scripts
+
+Lets stop using bash files and use v for everything
+
+example would be
+
+
+```go
+#!/usr/bin/env -S v -n -w -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run
+
+fn sh(cmd string) {
+	println('❯ ${cmd}')
+	print(execute_or_exit(cmd).output)
+}
+
+//super handy trick to go to where the file is
+abs_dir_of_script := dir(@FILE)
+
+
+sh('
+set -ex
+cd ${abs_dir_of_script}
+
+')
+
+//the $ shows its a compile time argument, will only put it compiled if linux
+$if !linux {
+	println('AM IN LINUX')
+}
+
+```
+
+## argument parsing
+
+```v
+#!/usr/bin/env -S v -n -w -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run
+
+import os
+import flag
+
+mut fp := flag.new_flag_parser(os.args)
+fp.application('compile.vsh')
+fp.version('v0.1.0')
+fp.description('Compile hero binary in debug or production mode')
+fp.skip_executable()
+
+prod_mode := fp.bool('prod', `p`, false, 'Build production version (optimized)')
+help_requested := fp.bool('help', `h`, false, 'Show help message')
+
+if help_requested {
+    println(fp.usage())
+    exit(0)
+}
+
+additional_args := fp.finalize() or {
+    eprintln(err)
+    println(fp.usage())
+    exit(1)
+}
+
+```

manual/best_practices/scripts/shebang.md

@@ -0,0 +1,14 @@
+# Shebang
+
+is the first line of a script, your os will use that one to get started.
+
+for V we use
+
+```bash
+#!/usr/bin/env -S v -n -w -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run
+```
+
+- -w no warnings 
+- -n ??? 
+
+This one is the fastest way how to run scripts, but not meant to run long running items

manual/best_practices/using_args_in_function.md

@@ -0,0 +1,32 @@
+# example how to use args in a function
+
+Below you can see a snippet which demonstrates how you have a params struct with path and git_url.
+
+- the params, makes sure you can call the function with
+  - path:...
+  - or git_url:... 
+- if git_url is used it will get the code from git and bring it in args.path
+- notice the trick where args is give immutable but then made mutable inside so we can change it later
+- git_pull means we will pull the code if the directory would already exist
+- git_reset means we ignore the files which are changed in the repo and will discard those local changes (dangerous)
+- reload: gets the git cache to be reloaded
+
+```go
+[params]
+pub struct TreeScannerArgs {
+pub mut:
+	name string = 'default' // name of tree
+	path string
+	git_url   string
+	git_reset bool
+	git_root  string
+	git_pull  bool
+}
+pub fn scan(args_ TreeScannerArgs) ! {
+	mut args := args_
+	if args.git_url.len > 0 {
+		args.path=gittools.code_get(coderoot:args.git_root,url:args.git_url,
+                pull:args.git_pull,reset:args.git_reset,reload:false)!
+	}
+
+```

manual/core/base.md

@@ -0,0 +1,87 @@
+
+# Base Context & Session
+
+Important section about how to create base objects which hold context an config mgmt.
+
+## Context
+
+A context is sort of sandbox in which we execute our scripts it groups the following
+
+- filesystem key value stor
+- logs
+- multiple sessions
+- gittools: gitstructure
+- redis client
+
+> more info see [context](context.md)
+
+## Session
+
+- each time we execute something with a client or other sal we do this as part of a session
+- a session can have a name as given by the developer or will be autocreated based on time
+
+> more info see [session](session.md)
+
+## Config Mgmt
+
+is done per instance of an object which inherits from BaseConfig.
+
+- see [base](base.md)
+- see [config](config.md)
+
+## KVS = FSDB
+
+there is a KVS attached to each context/session
+
+- see [kvs](kvs.md)
+
+
+# BaseConfig
+
+Clients, DALs, SAL's can inherit base
+
+
+```golang
+
+pub struct BaseConfig {
+pub mut:
+	session_ ?&Session 
+	instance    string
+}
+
+//how to use
+
+import freeflowuniverse.herolib.core.base
+
+pub struct B2Client {
+	base.BaseConfig
+pub mut:
+	someprop string
+}
+
+
+
+```
+
+## BaseConfig Methods
+
+This will give some super powers to each base inheritted class
+
+
+```v
+
+// return a session which has link to the actions and params on context and session level
+// the session also has link to dbfs (filesystem key val stor and gitstructure if relevant)
+//```
+// context             ?&Context      @[skip; str: skip]
+// session             ?&Session      @[skip; str: skip]
+// context_name        string = 'default'
+// session_name        string //default will be based on a date when run
+// interactive         bool = true //can ask questions, default on true
+//```
+pub fn (mut self BaseConfig) session(args PlayArgs) &Session
+
+pub fn (mut self BaseConfig) context() &Context
+
+
+```

manual/core/concepts/name_registry.md

@@ -0,0 +1,75 @@
+
+we are building a decentralized DNS system
+
+- the DNS system is stored in a filedb, which is a directory with subdirs where directory structure + file name, defines the key
+- we store pubkeys (32 bytes) and names (like dns names)- 
+- the public key gets mapped to a unique id per filedb dir
+- each person can register 1 or more names (max 12 characters, min 3)
+- these names are unique per repo and linked to the id of the public key
+- a name can be owned by min 1, max 5 public keys (names can be co-owned) = means max 4 bytes x 5 to identify which users own a name
+- we represent database for public keys and names as directory structures
+- database for public keys
+    - each pubkey as remembered in the file database in the repo on $repodir/keys gets a unique incremental key = int
+    - we have max 256 dirs and 256 files, where the name is first byte expressed as hex
+    - e.g. hex(999999) = 'f423f', this results in $repodir/keys/f4/23.txt
+    - in each txt file we \n separate the entries (each line is pubkey\n)
+    - 999999 -> f4/23 then f (remainder) gets converted back to int and this is the element in the list in 23.txt (the Xe line)
+    - this means max nr of dirs:65536, max nr of elements in file = 152 items, line separated
+    - this means it goes fast to process one txt file to retrieve relation between id and pubkey
+    - this db allows gast retrieval of pubkey based on int (unique per file db dir)
+    -  the order of the lines is never changed, new data always added so we keep unique id (int)
+- database for names
+  - names are ascii only with ofcourse '.' as separator
+  - names are 2 levels  e.g. kristof.belgium
+  - we hash the name md5, take first 2 chars as identifier for directory, the next 2 chars as text file with the names
+  - e.g. /data/repo1/names/aa/bb.txt  (aa would be first 2 chars of md5, bb next 2 chars of md5)
+  - names are in that bb.txt file (example), they are added as they come in
+  - the linenr is the unique id in that file, which means each name has unique id as follows
+    - aabb1 (position 1) would result to: aabb -> int + 1, e.g. position 999 would be hex2int(aabb)+999
+    - this would be the unique int
+  - per line we store the following: $name(lowercase, ascii):f423f,a4233:signature
+    - this means 2 pub keys linked to the name
+    - the link is done by an id (as described above, which can then be mapped back to pubkey)
+    - the signature is secp256k1 signature which can be verified by everyone who reads this file, only 1 of users need to sign
+      - the signature is on name+the id's of who owns the name (so we verify ownership)
+  - the order of the lines is never changed, new data always added so we keep unique id (int)
+
+now create the following python functions and implement above
+
+```python
+#register pubkey in the pubkey db, return the int
+def key_register(pubkey) -> int
+
+class NamePreparation:
+  name str
+  pubkeys []int #position of each pubkey in the pubkey db
+  signature []u8 #bytestr of the secp256k1 signature
+
+  #sign name + int's (always concatenated in same way) with given privkey
+  #the result is stored in signature on class
+  def sign(privkey):
+    #need to check that priv key given is part of the pubkeys
+
+  #return str representation which is $name:f423f,a4233,...:$signature
+  def str() -> str:
+      ...
+
+#name will be lowercased, trimmed space
+#max 1 dot in name (2 levels in DNS, top and 1 down)
+#signature is secp256k and will be verified in this function against all given pubkeys
+#the first pubkey need to have signed the name + 
+#returns the inique id of the name in this filedb repo
+def name_register(name:str,pubkeys:[]str,privkey:...) -> int:
+  #will use NamePreparation functionality
+  #str() will give the right str which is added as newline to the right file in the filedb
+
+#find the name, NotFound exception when name not found, 
+#if verify on then will check the signature vs the first pubkey of the list
+def name_get(id:int,verify:bool=True) -> str:
+
+def key_get(id:int) -> PubKey:
+
+
+
+```
+

manual/core/concepts/objects.md

@@ -0,0 +1,32 @@
+
+
+## rootobject with config
+
+example of 3 methods each of such rootobjects need to have
+
+```golang
+
+pub fn (mut c Context) str() string {
+	return c.heroscript() or {"BUG: can't represent the object properly, I try raw.\n$c"}
+}
+
+fn (mut c Context) str2() string {
+	return "cid:${c.cid} name:${c.name} " or {"BUG: can't represent the context properly, I try raw"}
+}
+
+//if executed needs to redefine this object
+pub fn (mut c Context) heroscript() !string {
+	mut out:="!!core.context_define ${c.str2()}\n"
+	mut params:=c.params()!
+	if ! params.empty(){
+		out+="\n!!core.context_params guid:${c.guid()}\n"
+		out+=params.heroscript()+"\n"
+	}
+	return out
+}
+
+//needs to be unique for universe
+pub fn (mut c Context) guid() string {
+	return "${c.cid}:${c.name}"
+}
+```

manual/core/concepts/sid.md

@@ -0,0 +1,69 @@
+
+# sid = Smart ID
+
+- format:
+  - smart id, is 3 to 6 letters, 0...z
+- the rid,cid and id are all smart id's
+- sid's are unique per circle
+- sid's can be converted to int easily
+
+## gid = Global ID
+
+Identifies an object in unique way on global level, normally not needed in heroscript, because heroscript most often executes in context of a circle
+
+- gid = rid.cid.oid
+  - rid = region id (regional identifier on which circle is defined), often not used today
+  - cid = circle id
+  - id = object id
+- each of above id's are smart id's
+
+
+The following are valid representations
+
+- '$rid.$cid.$id'
+- '$cid.$id' if rid is known
+- '$id' if rid and cid are known
+
+## automatically fill in 
+
+```golang
+!circle_role.define
+  id:'***' //means will be filled in automatically, unique per circle
+  name:'vpsales'      
+	circle:'tftech' //can be id of circle or name
+	role:'stakeholder' 
+```
+
+## code
+
+```golang
+pub struct SmartId {
+pub mut:
+	rid 		     string //regional id
+	cid          string //link to circle
+	id          string //content id
+}
+```
+
+## sid's can address the world
+
+- each object can be addressed by means of 3 smart id's
+    - $smartid_region (e.g. regional internet)
+    - $smartid_circle
+    - $smartid_object
+- object is any of the object types e.g. issue, story, ...
+- each object is identified as 
+    - $smartid_region.$smartid_circle.$smartid_object
+    - $smartid_circle.$smartid_object (will enherit the id from the region we are operating on)
+    - $smartid_object  (will enherit region and circle from the circle we are operating on)
+- smart id is
+    - 2 to 6 times [a...z|0...9]
+    - size to nr of objects
+        - 2 -> 26+10^2 = 1,296
+        - 3 -> 26+10^3 = 46,656
+        - 4 -> 26+10^4 = 1,679,616
+        - 5 -> 26+10^5 = 60,466,176
+        - 6 -> 26+10^6 = 2,176,782,336
+- a circle can be owned by 1 person or by a group (e.g. company, or administrators for e.g. blockchain DB)
+- e.g. 1a.e5q.9h would result to globally unique identifier 1a would be the region, e5q the circle, 9h is id of the obj in my circle
+

manual/core/context.md

@@ -0,0 +1,100 @@
+
+
+# Context
+
+## Get a context
+
+
+```js
+cid       string // rid.cid or just cid
+name      string // a unique name in cid
+params    paramsparser.Params
+redis     &redisclient.Redis
+dbcollection &dbfs.DBCollection  
+```    
+
+- cid is the unique id for a circle.
+- the default context is "default"
+- each context can have params attached to it, as can be set by the heroscripts
+- each context has a redis client (can be a different per context but normally not)
+- context db is a fs db (key value stor)
+
+
+```golang
+import freeflowuniverse.herolib.core.base
+
+
+struct ContextGetArgs {
+	name        string = "default" // a unique name in cid
+	interactive bool = true
+}
+
+//get context based on name, can overrule interactivity
+play.context_get(args_ ContextGetArgs) !Context
+
+
+```
+
+## Work with a context
+
+E.g. gitstructure is linked to a context
+
+```golang
+
+//return the gistructure as is being used in context
+fn (mut self Context) gitstructure() !&gittools.GitStructure 
+
+//reload gitstructure from filesystem
+fn (mut self Context) gitstructure_reload()
+
+//return the coderoot as is used in context
+fn (mut self Context) coderoot() !string 
+
+// load the context params from redis
+fn (mut self Context) load() !
+
+// save the params to redis
+fn (mut self Context) save() !
+
+```
+
+## get a custom DB from context
+
+```golang
+
+//get a unique db with a name per context
+fn (mut self Context) db_get(dbname string) !dbfs.DB 
+
+//get configuration DB is always per context
+fn (mut self Context) db_config_get() !dbfs.DB
+
+```
+
+## configure context through heroscript
+
+```js
+!!context.configure
+    name:'test'
+    coderoot:''
+    interactive:true
+```
+
+
+## Configure a context
+
+A context can get certain configuration e.g. params, coderoot, ... (in future encryption), configuration is optional.
+
+```golang
+
+// configure a context object
+// params:
+// ```
+// cid          string = "000" // rid.cid or cid allone
+// name         string // a unique name in cid
+// params       string
+// coderoot	 string
+// interactive  bool
+// ```
+fn context_configure(args_ ContextConfigureArgs) ! 
+
+```

manual/core/context_session_job.md

@@ -0,0 +1,26 @@
+# Circle
+
+
+- has a unique CID = circle id (is a SID)
+- has following components
+  - context 
+    - manages a state for one specific context
+    - has a name and unique cid, and is linked to 1 circle (there can be more than 1 in a circle)
+    - has params 
+    - has todo checklist
+  - session
+    - linked to 1 context
+    - has unique id (int) linked to context
+    - can have a name (optional)
+    - is like a chat session, can be any series of actions
+    - each action once in needs to be executed becomes a job
+    - a job is linked to a heroscript, which is the physical representation of all the jobs (actions) which need to be executed, the heroscript is in order.
+    - each action done on session is stateless in memory (no mem usage), in other words can pass Session around without worrying about its internal state
+    - we use redis as backend to keep the state
+  - job
+    - linked to a session
+    - has incremental id, in relation to session
+    - is the execution of 1 specific action (heroscript action)
+    - it results in logs being collected
+    - it results in params being set on session level (only when WAL)
+    - TODO: needs to be implemented on job (Kristof)

manual/core/play.md

@@ -0,0 +1,35 @@
+# Play
+
+Important section about how to create base objects which hold context an config mgmt.
+
+## Context
+
+A context is sort of sandbox in which we execute our scripts it groups the following
+
+- filesystem key value stor
+- logs
+- multiple sessions
+- gittools: gitstructure
+- redis client
+
+> more info see [context](context.md)
+
+## Session
+
+- each time we execute a playbook using heroscript we do it in a session
+- a session can have a name as given by the developer or will be autocreated based on time
+
+> more info see [session](session.md)
+
+## Config Mgmt
+
+is done per instance of an object which inherits from BaseConfig.
+
+- see [base](base.md)
+- see [config](config.md)
+
+## KVS
+
+there is a KVS attached to each context/session
+
+- see [kvs](kvs.md)

manual/core/session.md

@@ -0,0 +1,82 @@
+## play.session
+
+```js
+name                string // unique id for session (session id), can be more than one per context
+plbook              playbook.PlayBook //is how heroscripts are being executed
+interactive         bool = true
+params              paramsparser.Params
+start               ourtime.OurTime
+end                 ourtime.OurTime
+context             Context        //link back to the context
+```
+
+### **The PlayArgs:**
+
+- context             ?&Context   
+- session             ?&Session   
+- context_name        string = 'default'
+- session_name        string        //default will be based on a date when run
+- interactive         bool = true   //can ask questions, default on true
+- coderoot            string        //this will define where all code is checked out
+- playbook_url        string        //url of heroscript to get and execute in current context
+- playbook_path       string        //path of heroscript to get and execute
+- playbook_text       string        //heroscript to execute
+
+```golang
+import freeflowuniverse.herolib.core.base
+import freeflowuniverse.herolib.develop.gittools
+
+mut session:=play.session_new(
+    coderoot:'/tmp/code'
+    interactive:true
+)!
+
+//THE next could be in a module which we call
+
+pub fn play_git(mut session Session) ! {
+	for mut action in session.plbook.find(filter:'gittools.*')! {
+		mut p := action.params
+		mut repo := p.get_default('repo', '')!
+        ... do whatever is required to 
+	}
+}
+
+
+```
+
+
+### use playbook
+
+```golang
+
+// add playbook heroscript (starting from path, text or git url)
+//```
+// path    string
+// text    string
+// prio    int = 99
+// url     string
+//```	
+fn (mut session Session) playbook_add(args_ PLayBookAddArgs) !
+
+//show the sesstion playbook as heroscript
+fn (mut session Session) heroscript()
+
+// add priorities for the playbook, normally more internal per module
+fn (mut self Session) playbook_priorities_add(prios map[int]string)
+
+
+```
+
+### use the kvs database
+
+is stored on filesystem
+
+```golang
+
+// get db of the session, is unique per session
+fn (mut self Session) db_get() !dbfs.DB {
+
+// get the db of the config, is unique per context
+fn (mut self Session) db_config_get() !dbfs.DB {
+
+```

manual/documentation/docextractor.md

@@ -0,0 +1,17 @@
+# doc extractor
+
+is a python tool to help us to get .md files into our manual
+
+copies all readme.md files from the different lib directors to
+
+- e.g. $crystallib/manual/libreadme/installers_sysadmintools_actrunner.md 
+- note the name has the location inside of where info came from
+
+this allows us to make manual and to copy information from the readme's which are in library
+
+to run
+
+```bash
+~/code/github/freeflowuniverse/crystallib/tools/doc_extractor/extractor.sh
+```
+