diff --git a/aiprompts/ai_core/core_params.md b/aiprompts/ai_core/core_params.md index f201874a..28583c3c 100644 --- a/aiprompts/ai_core/core_params.md +++ b/aiprompts/ai_core/core_params.md @@ -5,7 +5,7 @@ works very well in combination with heroscript ## How to get the paramsparser ```v -import freeflowuniverse.crystallib.data.paramsparser +import freeflowuniverse.herolib.data.paramsparser // Create new params from text params := paramsparser.new("color:red size:'large' priority:1 enable:true")! diff --git a/aiprompts/ai_core/osal_os_system_tools.md b/aiprompts/ai_core/osal_os_system_tools.md index 355e3b16..b164cff7 100644 --- a/aiprompts/ai_core/osal_os_system_tools.md +++ b/aiprompts/ai_core/osal_os_system_tools.md @@ -4,7 +4,7 @@ import as ```vlang -import freeflowuniverse.crystallib.osal +import freeflowuniverse.herolib.osal osal.ping... @@ -72,14 +72,14 @@ mut pm:=process.processmap_get()? info returns like: ```json -}, freeflowuniverse.crystallib.process.ProcessInfo{ +}, freeflowuniverse.herolib.process.ProcessInfo{ cpu_perc: 0 mem_perc: 0 cmd: 'mc' pid: 84455 ppid: 84467 rss: 3168 - }, freeflowuniverse.crystallib.process.ProcessInfo{ + }, freeflowuniverse.herolib.process.ProcessInfo{ cpu_perc: 0 mem_perc: 0 cmd: 'zsh -Z -g' diff --git a/aiprompts/ui console chalk.md b/aiprompts/ui console chalk.md index f2ec8e2b..d1bd2384 100644 --- a/aiprompts/ui console chalk.md +++ b/aiprompts/ui console chalk.md @@ -8,7 +8,7 @@ Chalk offers functions:- `console.color_fg(text string, color string)` - To chan Example: ```vlang -import freeflowuniverse.crystallib.ui.console +import freeflowuniverse.herolib.ui.console # basic usage println('I am really ' + console.color_fg('happy', 'green')) diff --git a/aiprompts/ui console.md b/aiprompts/ui console.md index 5cdba86c..cd9e7c50 100644 --- a/aiprompts/ui console.md +++ b/aiprompts/ui console.md @@ -5,7 +5,7 @@ has mechanisms to print better to console, see the methods below import as ```vlang -import freeflowuniverse.crystallib.ui.console +import freeflowuniverse.herolib.ui.console ``` diff --git a/aiprompts/vshell example script instructions.md b/aiprompts/vshell example script instructions.md index 5da015d0..d87fa9d9 100644 --- a/aiprompts/vshell example script instructions.md +++ b/aiprompts/vshell example script instructions.md @@ -6,14 +6,14 @@ this is how we want example scripts to be, see the first line ```vlang #!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run -import freeflowuniverse.crystallib.installers.sysadmintools.daguserver +import freeflowuniverse.herolib.installers.sysadmintools.daguserver mut ds := daguserver.get()! println(ds) ``` -the files are in ~/code/github/freeflowuniverse/crystallib/examples for crystallib +the files are in ~/code/github/freeflowuniverse/herolib/examples for herolib ## important instructions diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..0411a911 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,50 @@ +# HeroLib Examples + +This repository contains examples and utilities for working with HeroLib, a comprehensive library for V language. + +## Sync Do Script + +The `sync_do.sh` script is a utility for development that: + +- Synchronizes the local HeroLib codebase with a remote server +- Uses rsync to efficiently transfer only changed files +- Automatically connects to a tmux session on the remote server +- Helps maintain development environment consistency + +## Examples Structure + +The examples directory demonstrates various capabilities of HeroLib: + +- **builder/**: Examples of builder patterns and remote execution +- **core/**: Core functionality examples including configuration, database operations, and API integrations +- **data/**: Data handling examples including encryption and encoding +- **develop/**: Development tools including git integration and OpenAI examples +- **hero/**: Hero-specific implementations and API examples +- **installers/**: Various installation scripts for different tools and services +- **lang/**: Language integration examples (e.g., Python) +- **osal/**: Operating system abstraction layer examples +- **threefold/**: ThreeFold Grid related examples and utilities +- **tools/**: Utility examples for imagemagick, tmux, etc. +- **ui/**: User interface examples including console and telegram +- **virt/**: Virtualization examples for Docker, Lima, Windows, etc. +- **webtools/**: Web-related tools and utilities + +## V Script Requirements + +When creating V scripts (.vsh files), always use the following shebang: + +```bash +#!/usr/bin/env -S v -n -w -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run +``` + +This shebang ensures: + +- Direct execution of V shell scripts without needing to use the V command +- No main() function requirement in .vsh files +- Proper compilation flags and settings +- OpenSSL support enabled +- Global variables enabled +- TCC compiler usage +- No retry compilation + +These examples serve as practical demonstrations and reference implementations for various HeroLib features and integrations. diff --git a/examples/builder/remote_executor/README.md b/examples/builder/remote_executor/README.md new file mode 100644 index 00000000..05e6b8ba --- /dev/null +++ b/examples/builder/remote_executor/README.md @@ -0,0 +1,74 @@ +# Remote Executor Example + +This example demonstrates how to compile and execute V code remotely using SSH. + +It shows a practical implementation of the herolib builder's remote execution capabilities, its good for debugging. + +## Components + +### `toexec.v` + +A V program that demonstrates remote execution of system operations: + +- Uses herolib's osal and installer modules +- Currently configured to uninstall brew as an example operation +- Can be modified to execute any remote system commands + +> important the source & target system needs to be same architecture + +### `run.sh` + +A bash script that: +1. Compiles the V program +2. Copies it to a remote machine using SCP +3. Executes it remotely using SSH + +## Prerequisites + +1. SSH access to the remote machine +2. The `SECRET` environment variable must be set +3. V compiler installed locally + +## Configuration + +The `run.sh` script uses the following default configuration: + +```bash +remote_user='despiegk' +remote_host='192.168.99.1' +remote_path='/Users/despiegk/hero/bin/toexec' +remote_port='2222' +``` + +Modify these values to match your remote system configuration. + +## Usage + +1. Set the required environment variable: +```bash +export SECRET=your_secret_value +``` + +2. Make the script executable: +```bash +chmod +x run.sh +``` + +3. Run the script: +```bash +./run.sh +``` + +## Integration with Builder + +This example demonstrates practical usage of the herolib builder module's remote execution capabilities. For more complex implementations, see the builder documentation in `lib/builder/readme.md`. + +The builder module provides a more structured way to manage remote nodes and execute commands: + +```v +import freeflowuniverse.herolib.builder +mut b := builder.new()! +mut n := b.node_new(ipaddr:"user@host:port")! +// Execute commands on the remote node +``` + diff --git a/examples/builder/remote_executor/run.sh b/examples/builder/remote_executor/run.sh new file mode 100755 index 00000000..0d162c9a --- /dev/null +++ b/examples/builder/remote_executor/run.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +# Check if the SECRET environment variable is set +if [ -z "$SECRET" ]; then + echo "Error: SECRET is not set." + exit 1 +fi + +cd "$(dirname "$0")" + +v -n -w -enable-globals toexec.v + + # Specify the local file to be copied and the remote destination +local_file='toexec' # Replace with the path to your local file +remote_user='despiegk' +remote_host='192.168.99.1' +remote_path='/Users/despiegk/hero/bin/toexec' +remote_port='2222' + + +scp -P ${remote_port} "${local_file}" "${remote_user}@${remote_host}:${remote_path}" +ssh -t -p ${remote_port} "${remote_user}@${remote_host}" -A "/bin/zsh -c 'source ~/.zshrc && ${remote_path}' && echo 'DONE'" \ No newline at end of file diff --git a/examples/builder/remote_executor/toexec.v b/examples/builder/remote_executor/toexec.v new file mode 100644 index 00000000..7d06c678 --- /dev/null +++ b/examples/builder/remote_executor/toexec.v @@ -0,0 +1,20 @@ +module main + +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.installers.base + +fn do() ! { + //base.uninstall_brew()! + //println("something") + if osal.is_osx() { + println('IS OSX') + } + + // mut job2 := osal.exec(cmd: 'ls /')! + // println(job2) + +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/builder/simple.vsh b/examples/builder/simple.vsh new file mode 100755 index 00000000..08ae31d2 --- /dev/null +++ b/examples/builder/simple.vsh @@ -0,0 +1,46 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.builder +import freeflowuniverse.herolib.core.pathlib +import os + + +fn do1() ! { + mut b := builder.new()! + mut n := b.node_new(ipaddr: 'root@195.192.213.2')! + + n.upload(source: myexamplepath, dest: '/tmp/myexamplepath2')! + n.download(source: '/tmp/myexamplepath2', dest: '/tmp/myexamplepath2', delete: true)! +} + +fn do2() ! { + mut b := builder.new()! + mut n := b.node_local()! + + n.upload(source: myexamplepath, dest: '/tmp/myexamplepath3', delete: true)! + + // lets now put something in examplepath3, which should be deleted + n.file_write('/tmp/myexamplepath3/something', 'something')! + r := n.file_read('/tmp/myexamplepath3/something')! + assert r == 'something' + + mut p2 := pathlib.get_dir(path: '/tmp/myexamplepath2')! // needs to exist, and is a dir + mut p3 := pathlib.get_dir(path: '/tmp/myexamplepath3')! + h2 := p2.md5hex()! + mut h3 := p3.md5hex()! + assert !(h2 == h3) + + n.upload(source: '/tmp/myexamplepath2', dest: '/tmp/myexamplepath3', delete: true)! + + // now hash should be the same, hashes work over all files in a dir + // its a good trick to compare if 2 directories are the same + h3 = p3.md5hex()! + assert h2 == h3 + + // there is also a size function, this one is in KByte + size := p3.size_kb() or { 0 } + println('size: ${size} KB') +} + +do1() +do2() diff --git a/examples/builder/simple_ip4.vsh b/examples/builder/simple_ip4.vsh new file mode 100755 index 00000000..b3c9aeeb --- /dev/null +++ b/examples/builder/simple_ip4.vsh @@ -0,0 +1,17 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.builder +import freeflowuniverse.herolib.core.pathlib +import os + +mut b := builder.new()! +mut n := b.node_new(ipaddr: 'root@51.195.61.5')! +// mut n := b.node_new(ipaddr: 'info.ourworld.tf')! + +println(n) + +r:=n.exec(cmd:"ls /")! +println(r) + +// n.upload(source: myexamplepath, dest: '/tmp/myexamplepath2')! +// n.download(source: '/tmp/myexamplepath2', dest: '/tmp/myexamplepath2', delete: true)! diff --git a/examples/builder/simple_ip6.vsh b/examples/builder/simple_ip6.vsh new file mode 100755 index 00000000..3c7a5a5d --- /dev/null +++ b/examples/builder/simple_ip6.vsh @@ -0,0 +1,14 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.builder +import freeflowuniverse.herolib.core.pathlib +import os + +mut b := builder.new()! +mut n := b.node_new(ipaddr: 'root@302:1d81:cef8:3049:ad01:796d:a5da:9c6')! + +r:=n.exec(cmd:"ls /")! +println(r) + +// n.upload(source: myexamplepath, dest: '/tmp/myexamplepath2')! +// n.download(source: '/tmp/myexamplepath2', dest: '/tmp/myexamplepath2', delete: true)! diff --git a/examples/crystallib.code-workspace b/examples/crystallib.code-workspace new file mode 100644 index 00000000..1348a966 --- /dev/null +++ b/examples/crystallib.code-workspace @@ -0,0 +1,20 @@ +{ + "folders": [ + { + "path": "../lib" + }, + { + "path": "../aiprompts" + }, + { + "path": "../research" + }, + { + "path": "../examples" + }, + { + "path": "../cli" + } + ], + "settings": {} +} \ No newline at end of file diff --git a/examples/data/encoder.vsh b/examples/data/encoder.vsh new file mode 100755 index 00000000..deac50c9 --- /dev/null +++ b/examples/data/encoder.vsh @@ -0,0 +1,53 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.data.encoder +import crypto.ed25519 +import freeflowuniverse.herolib.ui.console + +struct AStruct { +mut: + items []string + nr int + privkey []u8 +} + +_, privkey := ed25519.generate_key()! +mut a := AStruct{ + items: ['a', 'b'] + nr: 10 + // privkey: []u8{len: 5, init: u8(0xf8)} + privkey: privkey +} + +// do encoding +mut e := encoder.new() +e.add_list_string(a.items) +e.add_int(a.nr) +e.add_bytes(privkey) + +console.print_debug('${e.data}') + +// do decoding +mut d := encoder.decoder_new(e.data) +mut aa := AStruct{} +aa.items = d.get_list_string() +aa.nr = d.get_int() +aa.privkey = d.get_bytes() + +assert a == aa + + +a = AStruct{ + items: ['a', 'b'] + nr: 10 + privkey: []u8{len: 5, init: u8(0xf8)} +} + +serialize_data := encoder.encode(a)! + +r := encoder.decode[AStruct](serialize_data) or { + console.print_stderr('Failed to decode, error: ${err}') + return +} + +console.print_debug('${r}') diff --git a/examples/data/encrypt_decrypt.vsh b/examples/data/encrypt_decrypt.vsh new file mode 100755 index 00000000..0c629deb --- /dev/null +++ b/examples/data/encrypt_decrypt.vsh @@ -0,0 +1,16 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.crypt.aes_symmetric { decrypt, encrypt } +import freeflowuniverse.herolib.ui.console + +msg := 'my message'.bytes() +console.print_debug('${msg}') + +secret := '1234' +encrypted := encrypt(msg, secret) +console.print_debug('${encrypted}') + +decrypted := decrypt(encrypted, secret) +console.print_debug('${decrypted}') + +assert decrypted == msg \ No newline at end of file diff --git a/examples/data/heroencoder_example.vsh b/examples/data/heroencoder_example.vsh new file mode 100755 index 00000000..ed747612 --- /dev/null +++ b/examples/data/heroencoder_example.vsh @@ -0,0 +1,66 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run +import freeflowuniverse.herolib.data.encoderhero +import freeflowuniverse.herolib.core.base + +//this is docu at top +@[name:"teststruct " ; params] +pub struct TestStruct { +//this is docu at mid +pub mut: + id int @[hi] + descr string + secret string @[secret] + number int = 1 @[min:1 ;max:10] + yesno bool + liststr []string + listint []int + ss SubStruct + ss2 []SubStruct +} + +pub struct SubStruct { +pub mut: + color string + size int +} + +fn (self TestStruct) heroscript()!string { + mut out:="" + mut p := encoderhero.encode[TestStruct](self)! + // out += "!!hr.teststruct_define " + p.heroscript() + "\n" + // p = paramsparser.encode[SubStruct](self.ss)! + // p.set("teststruct_id",self.id.str()) + // out += "!!hr.substruct_define " + p.heroscript() + "\n" + // for ss2 in self.ss2{ + // p = paramsparser.encode[SubStruct](ss2)! + // p.set("teststruct_id",self.id.str()) + // out += "!!hr.substruct_item_define " + p.heroscript() + "\n" + // } + return p +} + + +mut t := TestStruct{ + id:100 + descr: ' + test + muliline + s + test + muliline + test + muliline + ' + number: 2 + yesno: true + liststr: ['one', 'two+two'] + listint: [1, 2] + ss:SubStruct{color:"red",size:10} +} +t.ss2<< SubStruct{color:"red1",size:11} +t.ss2<< SubStruct{color:"red2",size:12} + +println(t.heroscript()!) + +// t2:=p.decode[TestStruct]()! +// println(t2) diff --git a/examples/data/params/args/args_example.vsh b/examples/data/params/args/args_example.vsh new file mode 100755 index 00000000..1baf843f --- /dev/null +++ b/examples/data/params/args/args_example.vsh @@ -0,0 +1,31 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.data.paramsparser +import os + +const testpath = os.dir(@FILE) + '/data' + +ap := playbook.new(path: testpath)! + +mut test := map[string]string{} +test['root'] = 'YEH' +test['roott'] = 'YEH2' +for action in ap.actions { + // action.params.replace(test) + mut p := action.params + p.replace(test) + println(p) +} + +txt := ' + +this is a text \${aVAR} + +this is a text \${aVAR} + +\${A} + +' +// println(txt) +// println(params.regexfind(txt)) \ No newline at end of file diff --git a/examples/data/params/args/data/vars.md b/examples/data/params/args/data/vars.md new file mode 100644 index 00000000..97d397d1 --- /dev/null +++ b/examples/data/params/args/data/vars.md @@ -0,0 +1,8 @@ +```javascript +//will add an action can be https file, https git, scp, or local path +!!runner.recipe_add source:'{ROOT}/core/base0' aname:'{ROOTT}/base0' execute:1 + +//cannot define the name when we add a directory to it +!!runner.recipe_add source:'{ROOT}/core' execute:1 + +``` \ No newline at end of file diff --git a/examples/data/params/paramsfilter/.gitignore b/examples/data/params/paramsfilter/.gitignore new file mode 100644 index 00000000..9ef697fb --- /dev/null +++ b/examples/data/params/paramsfilter/.gitignore @@ -0,0 +1 @@ +paramsfilter \ No newline at end of file diff --git a/examples/data/params/paramsfilter/paramsfilter.vsh b/examples/data/params/paramsfilter/paramsfilter.vsh new file mode 100755 index 00000000..eadf2647 --- /dev/null +++ b/examples/data/params/paramsfilter/paramsfilter.vsh @@ -0,0 +1,45 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.data.paramsparser { Params, parse } +import time { Duration, sleep } + +totalnr := 1000000 + +// some performance tests +mut res := []Params{} +mut sw := time.new_stopwatch() +for i in 0 .. totalnr { + mut text := "arg${i} arg2 color:red${i} priority:'incredible' description:'with spaces, lets see if ok'" + mut p := parse(text) or { panic(err) } + res << p +} +sw.stop() +mut elapsed := sw.elapsed() +println(elapsed) +sw.restart() +incl_test := ['description:*see*'] +mut foundnr := 0 +for i in 0 .. totalnr { + mut p2 := res[i] + e := p2.filter_match(include: incl_test)! + f := p2.filter_match(include: ['arg100'])! + if f { + foundnr += 1 + } +} +assert foundnr == 1 +elapsed = sw.elapsed() +println(elapsed) +// sw.restart() + +mbused := 600.0 +bytesused := mbused * 1000 * 1000 +bytes_param := bytesused / totalnr +println('bytes used per param: ${bytes_param}') +println('nr of founds: ${foundnr}') + +// sleep(Duration(60 * time.minute)) + +// 600 bytes per params for 1m records +// TODO: not sure needs to be redone +// takes 0.9 sec to walk over 1million records diff --git a/examples/data/resp/resp_example.vsh b/examples/data/resp/resp_example.vsh new file mode 100755 index 00000000..8ad48afb --- /dev/null +++ b/examples/data/resp/resp_example.vsh @@ -0,0 +1,20 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.data.resp +import crypto.ed25519 + +mut b := resp.builder_new() +b.add(resp.r_list_string(['a', 'b'])) +b.add(resp.r_int(10)) +b.add(resp.r_ok()) +// to get some binary +pubkey, privkey := ed25519.generate_key()! + +b.add(resp.r_bytestring(privkey)) + +// b.data now has the info as binary data +// println(b.data) +println(b.data.bytestr()) + +lr := resp.decode(b.data)! +println(lr) diff --git a/examples/develop/gittools/.gitignore b/examples/develop/gittools/.gitignore new file mode 100644 index 00000000..d2ac1d7e --- /dev/null +++ b/examples/develop/gittools/.gitignore @@ -0,0 +1 @@ +gittools_example diff --git a/examples/develop/gittools/example3.vsh b/examples/develop/gittools/example3.vsh new file mode 100755 index 00000000..1a0ec6c6 --- /dev/null +++ b/examples/develop/gittools/example3.vsh @@ -0,0 +1,25 @@ +#!/usr/bin/env -S v -cg -enable-globals run + +import os +import freeflowuniverse.herolib.develop.gittools +import freeflowuniverse.herolib.develop.performance + +mut silent := false + +coderoot := if 'CODEROOT' in os.environ() { + os.environ()['CODEROOT'] +} else {os.join_path(os.home_dir(), 'code')} + +mut gs := gittools.get()! +if coderoot.len > 0 { + //is a hack for now + gs = gittools.new(coderoot: coderoot)! +} + +mypath := gs.do( + recursive: true + cmd: 'list' +)! + +timer := performance.new('gittools') +timer.timeline() \ No newline at end of file diff --git a/examples/develop/gittools/gittools_example.vsh b/examples/develop/gittools/gittools_example.vsh new file mode 100755 index 00000000..becaffda --- /dev/null +++ b/examples/develop/gittools/gittools_example.vsh @@ -0,0 +1,19 @@ +#!/usr/bin/env -S v -cg -enable-globals run + +import freeflowuniverse.herolib.develop.gittools +import freeflowuniverse.herolib.osal +import time + + +mut gs_default := gittools.new()! + +println(gs_default) + +// // Initializes the Git structure with the coderoot path. +// coderoot := '/tmp/code' +// mut gs_tmo := gittools.new(coderoot: coderoot)! + +// // Retrieve the specified repository. +// mut repo := gs_default.get_repo(name: 'herolib')! + +// println(repo) diff --git a/examples/develop/gittools/gittools_example2.vsh b/examples/develop/gittools/gittools_example2.vsh new file mode 100755 index 00000000..bf46c25d --- /dev/null +++ b/examples/develop/gittools/gittools_example2.vsh @@ -0,0 +1,104 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.develop.gittools +import freeflowuniverse.herolib.osal +import time + +// Creates a new file in the specified repository path and returns its name. +fn create_new_file(repo_path string, runtime i64)! string { + coded_now := time.now().unix() + file_name := 'hello_world_${coded_now}.py' + println('Creating a new ${file_name} file.') + + // Create a new file in the repository. + osal.execute_silent("echo \"print('Hello, World!')\" > ${repo_path}/${file_name}")! + return file_name +} + +// Resets all configurations and caches if needed. +// gittools.cachereset()! + +// Initializes the Git structure with the coderoot path. +coderoot := '~/code' +mut gs_default := gittools.new(coderoot: coderoot)! + +// Retrieve the specified repository. +mut repo := gs_default.get_repo(name: 'repo3')! + +// In case we need to clone it, will clone the repo2 in a folder named repo3 +// mut repo := gs_default.get_repo(name: 'repo3' clone: true, url: 'https://github.com/Mahmoud-Emad/repo2.git')! + +runtime := time.now().unix() +branch_name := "branch_${runtime}" +tag_name := "tag_${runtime}" +repo_path := repo.get_path()! +mut file_name := create_new_file(repo_path, runtime)! + +// Create a new branch to add our changes on it. +// We can simply checkout to the newly created branch, but we need to test the checkout method functionalty. + +println('Creating a new \'${branch_name}\' branch...') +repo.create_branch(branch_name: branch_name, checkout: false) or { + error("Couldn't create branch due to: ${err}") +} + +// Checkout to the created branch +println('Checkout to \'${branch_name}\' branch...') +repo.checkout(branch_name: branch_name, pull: false) or { + error("Couldn't checkout to branch ${branch_name} due to: ${err}") +} + +// Check for changes and stage them if present. +if repo.has_changes()! { + println('Adding the changes...') + repo.add_changes() or { + error('Cannot add the changes due to: ${err}') + } +} + +// Check if a commit is needed and commit changes if necessary. +if repo.need_commit()! { + commit_msg := 'feat: Added ${file_name} file.' + println('Committing the changes, Commit message: ${commit_msg}.') + repo.commit(msg: commit_msg) or { + error('Cannot commit the changes due to: ${err}') + } +} + +// Push changes to the remote repository if necessary. +if repo.need_push()! { + println('Pushing the changes...') + repo.push() or { + error('Cannot push the changes due to: ${err}') + } +} + +if repo.need_pull()! { + println('Pulling the changes.') + repo.pull() or { + error('Cannot pull the changes due to: ${err}') + } +} + +// Checkout to the base branch +repo.checkout(checkout_to_base_branch: true, pull: true) or { + error("Couldn't checkout to branch ${branch_name} due to: ${err}") +} + +// Create a new tag and add some changes on it then push it to the remote. +println('Creating a new \'${tag_name}\' tag...') +repo.create_tag(tag_name: tag_name, checkout: false) or { + error("Couldn't create tag due to: ${err}") +} + +// Push the created tag. +println('Pushing the tag...') +repo.push(push_tag: true) or { + error('Cannot push the tag due to: ${err}') +} + +// Check if the created tag exists. +println('Check if the created tag exists...') +repo.is_tag_exists(tag_name: tag_name) or { + println("Tag isn't exists.") +} \ No newline at end of file diff --git a/examples/develop/juggler/README.md b/examples/develop/juggler/README.md new file mode 100644 index 00000000..c24d941f --- /dev/null +++ b/examples/develop/juggler/README.md @@ -0,0 +1,7 @@ +## Juggler Example + +This example demonstrates how juggler is able to trigger DAG's on a remote dagu server, upon webhook triggers from gitea. + +To run example: +- configure gitea webhook to call `trigger` endpoint in your locally running juggler server +- run `main.vsh` with the appropriate `repo_path` and `dagu server url` \ No newline at end of file diff --git a/examples/develop/juggler/hero/example.sh b/examples/develop/juggler/hero/example.sh new file mode 100644 index 00000000..601a44c6 --- /dev/null +++ b/examples/develop/juggler/hero/example.sh @@ -0,0 +1 @@ +hero run -u https://github.com/freeflowuniverse/herolib/tree/development_juggler/examples/develop/juggler/hero/playbook \ No newline at end of file diff --git a/examples/develop/juggler/hero/playbook/juggler.md b/examples/develop/juggler/hero/playbook/juggler.md new file mode 100644 index 00000000..42cb741e --- /dev/null +++ b/examples/develop/juggler/hero/playbook/juggler.md @@ -0,0 +1,14 @@ +!!juggler.configure + url: 'https://git.ourworld.tf/projectmycelium/itenv' + username: '' + password: '' + port: 8000 + +!!juggler.start + +!!caddy.add_reverse_proxy + from: ':8000' + to: juggler.protocol.me + +!!caddy.generate +!!caddy.start \ No newline at end of file diff --git a/examples/develop/juggler/hero_example.sh b/examples/develop/juggler/hero_example.sh new file mode 100644 index 00000000..669028ee --- /dev/null +++ b/examples/develop/juggler/hero_example.sh @@ -0,0 +1 @@ +hero juggler -u https://git.ourworld.tf/projectmycelium/itenv \ No newline at end of file diff --git a/examples/develop/juggler/v_example b/examples/develop/juggler/v_example new file mode 100755 index 00000000..9f97a839 Binary files /dev/null and b/examples/develop/juggler/v_example differ diff --git a/examples/develop/juggler/v_example.vsh b/examples/develop/juggler/v_example.vsh new file mode 100755 index 00000000..7b9136de --- /dev/null +++ b/examples/develop/juggler/v_example.vsh @@ -0,0 +1,29 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import os +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.develop.juggler +import veb + +osal.load_env_file('${os.dir(@FILE)}/.env')! + +mut j := juggler.configure( + url: 'https://git.ourworld.tf/projectmycelium/itenv' + username: os.getenv('JUGGLER_USERNAME') + password: os.getenv('JUGGLER_PASSWORD') + reset: true +)! + +spawn j.run(8000) +println(j.info()) + +for{} + +// TODO +// - automate caddy install/start +// - create server/caddy which only calls install & can set config file from path or url & restart (see dagu server) +// - get caddy config from the itenv through (simple driver) +// - caddy through startup manager, also for dagu +// - expose dagu UI over caddy & make sure we use secret +// - have heroscript starting from itenv to start a full env (with secret): 'hero juggler -i -s mysecret --dns juggler2.protocol.me ' +// - use domain name use https://github.com/Incubaid/dns/blob/main/protocol.me.lua over git ssh diff --git a/examples/develop/juggler/v_example2 b/examples/develop/juggler/v_example2 new file mode 100755 index 00000000..b89870ab Binary files /dev/null and b/examples/develop/juggler/v_example2 differ diff --git a/examples/develop/juggler/v_example2.vsh b/examples/develop/juggler/v_example2.vsh new file mode 100755 index 00000000..08ebda78 --- /dev/null +++ b/examples/develop/juggler/v_example2.vsh @@ -0,0 +1,21 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.sysadmin.startupmanager +import os + +mut sm := startupmanager.get()! +sm.start( + name: 'juggler' + cmd: 'hero juggler -secret planetfirst -u https://git.ourworld.tf/projectmycelium/itenv -reset true' + env: {'HOME': os.home_dir()} + restart: true +) or {panic('failed to start sm ${err}')} + +// TODO +// - automate caddy install/start +// - create server/caddy which only calls install & can set config file from path or url & restart (see dagu server) +// - get caddy config from the itenv through (simple driver) +// - caddy through startup manager, also for dagu +// - expose dagu UI over caddy & make sure we use secret +// - have heroscript starting from itenv to start a full env (with secret): 'hero juggler -i -s mysecret --dns juggler2.protocol.me ' +// - use domain name use https://github.com/Incubaid/dns/blob/main/protocol.me.lua over git ssh diff --git a/examples/develop/luadns/example.vsh b/examples/develop/luadns/example.vsh new file mode 100644 index 00000000..7a34ab4c --- /dev/null +++ b/examples/develop/luadns/example.vsh @@ -0,0 +1,28 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.develop.luadns + +fn main() { + mut lua_dns := luadns.load('https://github.com/Incubaid/dns') or { + eprintln('Failed to parse LuaDNS files: $err') + return + } + + lua_dns.set_domain('test.protocol.me', '65.21.132.119') or { + eprintln('Failed to set domain: $err') + return + } + + lua_dns.set_domain('example.protocol.me', '65.21.132.119') or { + eprintln('Failed to set domain: $err') + return + } + + for config in lua_dns.configs { + println(config) + } + + for config in lua_dns.configs { + println(config) + } +} \ No newline at end of file diff --git a/examples/develop/openai/.gitignore b/examples/develop/openai/.gitignore new file mode 100644 index 00000000..82c06e18 --- /dev/null +++ b/examples/develop/openai/.gitignore @@ -0,0 +1 @@ +openai_example \ No newline at end of file diff --git a/examples/develop/openai/openai_example.vsh b/examples/develop/openai/openai_example.vsh new file mode 100644 index 00000000..8f04cf6f --- /dev/null +++ b/examples/develop/openai/openai_example.vsh @@ -0,0 +1,76 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.clients.openai as op + +mut ai_cli := op.new()! +mut msg := []op.Message{} +msg << op.Message{ + role: op.RoleType.user + content: 'Say this is a test!' +} +mut msgs := op.Messages{ + messages: msg +} +res := ai_cli.chat_completion(op.ModelType.gpt_3_5_turbo, msgs)! +print(res) + +models := ai_cli.list_models()! + +model := ai_cli.get_model(models.data[0].id)! +print(model) +images_created := ai_cli.create_image(op.ImageCreateArgs{ + prompt: 'Calm weather' + num_images: 2 + size: op.ImageSize.size_512_512 + format: op.ImageRespType.url +})! +print(images_created) +images_updated := ai_cli.create_edit_image(op.ImageEditArgs{ + image_path: '/path/to/image.png' + mask_path: '/path/to/mask.png' + prompt: 'Calm weather' + num_images: 2 + size: op.ImageSize.size_512_512 + format: op.ImageRespType.url +})! +print(images_updated) +images_variatons := ai_cli.create_variation_image(op.ImageVariationArgs{ + image_path: '/path/to/image.png' + num_images: 2 + size: op.ImageSize.size_512_512 + format: op.ImageRespType.url +})! +print(images_variatons) + +transcription := ai_cli.create_transcription(op.AudioArgs{ + filepath: '/path/to/audio' +})! +print(transcription) + +translation := ai_cli.create_tranlation(op.AudioArgs{ + filepath: '/path/to/audio' +})! +print(translation) + +file_upload := ai_cli.upload_file(filepath: '/path/to/file.jsonl', purpose: 'fine-tune') +print(file_upload) +files := ai_cli.list_filess()! +print(files) +resp := ai_cli.create_fine_tune(training_file: file.id, model: 'curie')! +print(resp) + +fine_tunes := ai_cli.list_fine_tunes()! +print(fine_tunes) + +fine_tune := ai_cli.get_fine_tune(fine_tunes.data[0].id)! +print(fine_tune) + +moderations := ai_cli.create_moderation('Something violent', op.ModerationModel.text_moderation_latest)! +print(moderations) + +embeddings := ai_cli.create_embeddings( + input: ['sample embedding input'] + model: op.EmbeddingModel.text_embedding_ada +)! +print(embeddings) + diff --git a/examples/hero/alpine_example.vsh b/examples/hero/alpine_example.vsh index 5db4c894..eea4540b 100755 --- a/examples/hero/alpine_example.vsh +++ b/examples/hero/alpine_example.vsh @@ -1,7 +1,7 @@ #!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run -import freeflowuniverse.crystallib.hero.bootstrap +import freeflowuniverse.herolib.hero.bootstrap mut al:=bootstrap.new_alpine_loader() diff --git a/examples/hero/generation/blank_generation/example_1.vsh b/examples/hero/generation/blank_generation/example_1.vsh index 73133089..9cbdae21 100644 --- a/examples/hero/generation/blank_generation/example_1.vsh +++ b/examples/hero/generation/blank_generation/example_1.vsh @@ -1,6 +1,6 @@ #!/usr/bin/env -S v -w -n -enable-globals run -import freeflowuniverse.crystallib.hero.generation +import freeflowuniverse.herolib.hero.generation generation.generate_actor( name: 'Example' diff --git a/examples/hero/generation/blank_generation/example_2.vsh b/examples/hero/generation/blank_generation/example_2.vsh index 536e195b..1ecfc8e1 100644 --- a/examples/hero/generation/blank_generation/example_2.vsh +++ b/examples/hero/generation/blank_generation/example_2.vsh @@ -1,6 +1,6 @@ #!/usr/bin/env -S v -w -n -enable-globals run -import freeflowuniverse.crystallib.hero.generation +import freeflowuniverse.herolib.hero.generation generation.generate_actor( name: 'Example' diff --git a/examples/hero/generation/openapi_generation/example_actor/actor.v b/examples/hero/generation/openapi_generation/example_actor/actor.v index ad76d387..02bed1c5 100644 --- a/examples/hero/generation/openapi_generation/example_actor/actor.v +++ b/examples/hero/generation/openapi_generation/example_actor/actor.v @@ -1,8 +1,8 @@ module example_actor import os -import freeflowuniverse.crystallib.hero.baobab.actor {IActor, RunParams} -import freeflowuniverse.crystallib.web.openapi +import freeflowuniverse.herolib.hero.baobab.actor {IActor, RunParams} +import freeflowuniverse.herolib.web.openapi import time const openapi_spec_path = '${os.dir(@FILE)}/specs/openapi.json' diff --git a/examples/hero/openapi/actor.vsh b/examples/hero/openapi/actor.vsh index 776c5f2f..affb51a5 100755 --- a/examples/hero/openapi/actor.vsh +++ b/examples/hero/openapi/actor.vsh @@ -6,9 +6,9 @@ import veb import json import x.json2 import net.http -import freeflowuniverse.crystallib.web.openapi {Server, Context, Request, Response} -import freeflowuniverse.crystallib.hero.processor {Processor, ProcedureCall, ProcedureResponse, ProcessParams} -import freeflowuniverse.crystallib.clients.redisclient +import freeflowuniverse.herolib.web.openapi {Server, Context, Request, Response} +import freeflowuniverse.herolib.hero.processor {Processor, ProcedureCall, ProcedureResponse, ProcessParams} +import freeflowuniverse.herolib.clients.redisclient @[heap] struct Actor { diff --git a/examples/hero/openapi/server.vsh b/examples/hero/openapi/server.vsh index 57262925..93ee215a 100755 --- a/examples/hero/openapi/server.vsh +++ b/examples/hero/openapi/server.vsh @@ -6,10 +6,10 @@ import veb import json import x.json2 {Any} import net.http -import freeflowuniverse.crystallib.data.jsonschema {Schema} -import freeflowuniverse.crystallib.web.openapi {Server, Context, Request, Response} -import freeflowuniverse.crystallib.hero.processor {Processor, ProcedureCall, ProcedureResponse, ProcessParams} -import freeflowuniverse.crystallib.clients.redisclient +import freeflowuniverse.herolib.data.jsonschema {Schema} +import freeflowuniverse.herolib.web.openapi {Server, Context, Request, Response} +import freeflowuniverse.herolib.hero.processor {Processor, ProcedureCall, ProcedureResponse, ProcessParams} +import freeflowuniverse.herolib.clients.redisclient const spec_path = '${os.dir(@FILE)}/data/openapi.json' const spec_json = os.read_file(spec_path) or { panic(err) } diff --git a/examples/installers/.gitignore b/examples/installers/.gitignore new file mode 100644 index 00000000..f92ee2b8 --- /dev/null +++ b/examples/installers/.gitignore @@ -0,0 +1,5 @@ +caddy +gitea +installers +postgresql +mycelium diff --git a/examples/installers/actrunner.vsh b/examples/installers/actrunner.vsh new file mode 100755 index 00000000..21650687 --- /dev/null +++ b/examples/installers/actrunner.vsh @@ -0,0 +1,7 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.sysadmintools.actrunner +import freeflowuniverse.herolib.installers.virt.herocontainers + +actrunner.install()! +//herocontainers.start()! \ No newline at end of file diff --git a/examples/installers/conduit.vsh b/examples/installers/conduit.vsh new file mode 100755 index 00000000..43704eda --- /dev/null +++ b/examples/installers/conduit.vsh @@ -0,0 +1,6 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.fediverse.conduit + +conduit.install()! + diff --git a/examples/installers/coredns.vsh b/examples/installers/coredns.vsh new file mode 100755 index 00000000..cd70dfe2 --- /dev/null +++ b/examples/installers/coredns.vsh @@ -0,0 +1,7 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + + +import freeflowuniverse.herolib.installers.infra.coredns as coredns_installer + + +coredns_installer.install()! diff --git a/examples/installers/dagu.vsh b/examples/installers/dagu.vsh new file mode 100755 index 00000000..9ad1759f --- /dev/null +++ b/examples/installers/dagu.vsh @@ -0,0 +1,16 @@ +#!/usr/bin/env -S v -cg -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run +// #!/usr/bin/env -S v -n -cg -w -enable-globals run + +import freeflowuniverse.herolib.installers.sysadmintools.daguserver +import freeflowuniverse.herolib.installers.infra.zinit + +//make sure zinit is there and running, will restart it if needed +mut z:=zinit.get()! +z.destroy()! +z.start()! + +// mut ds := daguserver.get()! +// ds.destroy()! +// ds.start()! + +// println(ds) diff --git a/examples/installers/dagu_server.vsh b/examples/installers/dagu_server.vsh new file mode 100755 index 00000000..f20604ad --- /dev/null +++ b/examples/installers/dagu_server.vsh @@ -0,0 +1,61 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.sysadmintools.daguserver + +//will call the installer underneith + +mut dserver:=daguserver.new()! +dserver.install()! +dserver.restart()! + +println("DAGU installed & running") + +mut dagucl:=dserver.client()! + + +// name string // The name of the DAG, which is optional. The default name is the name of the file. +// description ?string // A brief description of the DAG. +// tags ?string // Free tags that can be used to categorize DAGs, separated by commas. +// env ?map[string]string // Environment variables that can be accessed by the DAG and its steps. +// restart_wait_sec ?int // The number of seconds to wait after the DAG process stops before restarting it. +// hist_retention_days ?int // The number of days to retain execution history (not for log files). +// delay_sec ?int // The interval time in seconds between steps. +// max_active_runs ?int // The maximum number of parallel running steps. +// max_cleanup_time_sec ?int // The maximum time to wait after sending a TERM signal to running steps before killing them. + + +mut mydag:=dagucl.dag_new( + nameswhere:"test11" +) + +// nr int @[required] +// name string // The name of the step. +// description string // A brief description of the step. +// dir string // The working directory for the step. +// command string // The command and parameters to execute. +// stdout string // The file to which the standard output is written. +// output ?string // The variable to which the result is written. +// script ?string // The script to execute. +// signal_on_stop ?string // The signal name (e.g., SIGINT) to be sent when the process is stopped. +// continue_on_error bool +// depends string +// retry_nr int = 3 +// retry_interval int = 5 + +mydag.step_add( + script : "ls /tmp" + retry_interval:1 + retry_nr:3 + )! + +mydag.step_add( + script : "ls /root" + retry_interval:1 + retry_nr:3 + )! + + +dagresult:=dagucl.dag_register(mydag,start:true)! +println(dagresult) + +println("DAGU should have new steps") diff --git a/examples/installers/gitea.vsh b/examples/installers/gitea.vsh new file mode 100755 index 00000000..f30eb6f8 --- /dev/null +++ b/examples/installers/gitea.vsh @@ -0,0 +1,13 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.gitea + +mut g := gitea.new( + passwd: '123' + postgresql_path: '/tmp/db' + postgresql_reset: true + domain: 'git.meet.tf' + appname: 'ourworld' +)! +// postgresql will be same passwd +g.restart()! diff --git a/examples/installers/griddriver.vsh b/examples/installers/griddriver.vsh new file mode 100755 index 00000000..2595bdab --- /dev/null +++ b/examples/installers/griddriver.vsh @@ -0,0 +1,5 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.threefold.griddriver +mut griddriver_installer := griddriver.get()! +griddriver_installer.install()! diff --git a/examples/installers/hero_install.vsh b/examples/installers/hero_install.vsh new file mode 100755 index 00000000..6dc5d5c9 --- /dev/null +++ b/examples/installers/hero_install.vsh @@ -0,0 +1,9 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.lang.vlang +import freeflowuniverse.herolib.installers.sysadmintools.daguserver +import freeflowuniverse.herolib.installers.sysadmintools.b2 as b2_installer + +vlang.v_analyzer_install()! +daguserver.new()! //will install & start a daguserver + diff --git a/examples/installers/herocontainers.vsh b/examples/installers/herocontainers.vsh new file mode 100755 index 00000000..9d7125cd --- /dev/null +++ b/examples/installers/herocontainers.vsh @@ -0,0 +1,15 @@ +#!/usr/bin/env -S v -cg -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.installers.lang.golang + +import freeflowuniverse.herolib.installers.virt.podman as podman_installer +import freeflowuniverse.herolib.installers.virt.buildah as buildah_installer + +mut podman_installer0:= podman_installer.get()! +mut buildah_installer0:= buildah_installer.get()! + +//podman_installer0.destroy()! //will remove all + +podman_installer0.install()! +buildah_installer0.install()! \ No newline at end of file diff --git a/examples/installers/installers.vsh b/examples/installers/installers.vsh new file mode 100755 index 00000000..4634605e --- /dev/null +++ b/examples/installers/installers.vsh @@ -0,0 +1,29 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.data.dbfs +import freeflowuniverse.herolib.installers.lang.vlang +import freeflowuniverse.herolib.installers.db.redis as redis_installer +import freeflowuniverse.herolib.installers.infra.coredns as coredns_installer +import freeflowuniverse.herolib.installers.sysadmintools.daguserver as dagu_installer +import freeflowuniverse.herolib.installers.sysadmintools.b2 as b2_installer +import freeflowuniverse.herolib.installers.net.mycelium as mycelium_installer +import freeflowuniverse.herolib.osal.screen +// import freeflowuniverse.herolib.osal + +// redis_installer.new()! +// dagu_installer.install(passwd:"1234",secret:"1234",restart:true)! + +// coredns_installer.install()! +mycelium_installer.install()! +// mycelium_installer.restart()! + +// mut screens:=screen.new()! +// println(screens) + + +// dagu_installer.check(secret:"1234")! + + +vlang.v_analyzer_install()! + +// b2_installer.install()! \ No newline at end of file diff --git a/examples/installers/mycelium.vsh b/examples/installers/mycelium.vsh new file mode 100755 index 00000000..adb52b72 --- /dev/null +++ b/examples/installers/mycelium.vsh @@ -0,0 +1,4 @@ +#!/usr/bin/env -S v -cg -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.net.mycelium as mycelium_installer +mycelium_installer.start()! diff --git a/examples/installers/postgresql.vsh b/examples/installers/postgresql.vsh new file mode 100755 index 00000000..28f16101 --- /dev/null +++ b/examples/installers/postgresql.vsh @@ -0,0 +1,15 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -cg -d use_openssl -enable-globals run + + +import time + +import freeflowuniverse.herolib.installers.db.postgresql + +mut db:= postgresql.get()! + +// db.destroy()! +db.start()! + +// db.db_create('my_new_db')! +// db.stop()! +// db.start()! \ No newline at end of file diff --git a/examples/installers/youki.vsh b/examples/installers/youki.vsh new file mode 100755 index 00000000..600de23e --- /dev/null +++ b/examples/installers/youki.vsh @@ -0,0 +1,10 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + + +import freeflowuniverse.herolib.installers.virt.youki + +mut youki_installer:= youki.get()! + +youki_installer.install()! + + diff --git a/examples/lang/python/.gitignore b/examples/lang/python/.gitignore new file mode 100644 index 00000000..662e0f97 --- /dev/null +++ b/examples/lang/python/.gitignore @@ -0,0 +1 @@ +pythonexample diff --git a/examples/lang/python/pythonexample.py b/examples/lang/python/pythonexample.py new file mode 100644 index 00000000..f9a4b309 --- /dev/null +++ b/examples/lang/python/pythonexample.py @@ -0,0 +1,22 @@ + +import json + + +for counter in range(1, @nrcount): # Loop from 1 to 10 + print(f"done_{counter}") + + +# Define a simple Python structure (e.g., a dictionary) +example_struct = { + "name": "John Doe", + "age": @nrcount, + "is_member": True, + "skills": ["Python", "Data Analysis", "Machine Learning"] +} + +# Convert the structure to a JSON string +json_string = json.dumps(example_struct, indent=4) + +# Print the JSON string +print("==RESULT==") +print(json_string) \ No newline at end of file diff --git a/examples/lang/python/pythonexample.vsh b/examples/lang/python/pythonexample.vsh new file mode 100755 index 00000000..b5f6dd5f --- /dev/null +++ b/examples/lang/python/pythonexample.vsh @@ -0,0 +1,34 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + + +import freeflowuniverse.herolib.lang.python +import json + + +pub struct Person { + name string + age int + is_member bool + skills []string +} + + +mut py:=python.new(name:'test')! //a python env with name test +//py.update()! +py.pip("ipython")! + + +nrcount:=5 +cmd:=$tmpl("pythonexample.py") + +mut res:="" +for i in 0..5{ + println(i) + res=py.exec(cmd:cmd)! + +} +//res:=py.exec(cmd:cmd)! + +person:=json.decode(Person,res)! +println(person) + diff --git a/examples/osal/.gitignore b/examples/osal/.gitignore new file mode 100644 index 00000000..f6779d07 --- /dev/null +++ b/examples/osal/.gitignore @@ -0,0 +1 @@ +lima_example \ No newline at end of file diff --git a/examples/osal/download/download_example.vsh b/examples/osal/download/download_example.vsh new file mode 100755 index 00000000..27b86965 --- /dev/null +++ b/examples/osal/download/download_example.vsh @@ -0,0 +1,14 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.osal { download } + +mut p := download( + url: 'https://cdnjs.cloudflare.com/ajax/libs/echarts/5.4.3/@name' + name: 'echarts.min.js' + reset: false + dest: '/tmp/@name' + minsize_kb: 1000 + maxsize_kb: 5000 +)! + +println(p) diff --git a/examples/osal/ping/ping_example.vsh b/examples/osal/ping/ping_example.vsh new file mode 100755 index 00000000..285700c7 --- /dev/null +++ b/examples/osal/ping/ping_example.vsh @@ -0,0 +1,7 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.osal { ping } + +assert ping(address: '338.8.8.8')! == .unknownhost +assert ping(address: '8.8.8.8')! == .ok +assert ping(address: '18.8.8.8')! == .timeout \ No newline at end of file diff --git a/examples/osal/ping/portforward.vsh b/examples/osal/ping/portforward.vsh new file mode 100755 index 00000000..20e84888 --- /dev/null +++ b/examples/osal/ping/portforward.vsh @@ -0,0 +1,11 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.builder + +// name string @[required] +// address string @[required] +// remote_port int @[required] + +builder.portforward_to_local(name:"holo1",address:"[302:1d81:cef8:3049:fbe1:69ba:bd8c:52ec]",remote_port:45579)! +builder.portforward_to_local(name:"holo2",address:"[302:1d81:cef8:3049:fbe1:69ba:bd8c:52ec]",remote_port:34639)! +builder.portforward_to_local(name:"holoui",address:"[302:1d81:cef8:3049:fbe1:69ba:bd8c:52ec]",remote_port:8282)! \ No newline at end of file diff --git a/examples/osal/process/process.v b/examples/osal/process/process.v new file mode 100644 index 00000000..1701ba70 --- /dev/null +++ b/examples/osal/process/process.v @@ -0,0 +1,20 @@ +module main + +import freeflowuniverse.herolib.osal + +fn do() ? { + if osal.is_osx() { + println('IS OSX') + } + + mut job2 := osal.exec(cmd: 'ls /')? + println(job2) + + // wont die, the result can be found in /tmp/execscripts + mut job := osal.exec(cmd: 'ls dsds', die: false)? + println(job) +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/osal/process/process_bash/processtest.v b/examples/osal/process/process_bash/processtest.v new file mode 100644 index 00000000..43a1b08a --- /dev/null +++ b/examples/osal/process/process_bash/processtest.v @@ -0,0 +1,64 @@ +module main + +import os +import time + +fn main() { + do1() or { panic(err) } +} + +fn do1() ! { + + mut p := os.new_process("/bin/bash") + p.set_work_folder("/tmp") + p.set_redirect_stdio() + p.use_stdio_ctl = true + p.use_pgroup = true + // p.set_args(['-i','-q']) + p.run() + // p.set_args("") + // time.sleep(100 * time.millisecond) + println( "alive: ${p.is_alive()}") + assert p.is_alive() + + defer { + p.wait() + p.close() + } + + // for { + // println(1) + // println(p.stdout_slurp()) + // println(2) + // println(p.stderr_slurp()) + // println(3) + // } + mut counter:=0 + for { + counter+=1 + println(counter) + out:=p.pipe_read(.stdout) or {""} + if out.len>0{ + println("o") + println(out) + } + err:=p.pipe_read(.stderr) or {""} + if err.len>0{ + println("e") + println(err) + } + time.sleep(100 * time.millisecond) + if counter==2{ + p.stdin_write("echo '111'\n") + // os.fd_close(p.stdio_fd[0]) + } + if counter==20{ + p.stdin_write("echo '2222'\n") + // os.fd_close(p.stdio_fd[0]) + } + } +} + + + + diff --git a/examples/osal/process/process_python/processtest.v b/examples/osal/process/process_python/processtest.v new file mode 100644 index 00000000..555a66a4 --- /dev/null +++ b/examples/osal/process/process_python/processtest.v @@ -0,0 +1,131 @@ +module main + +import os +import time + +fn main() { + do1() or { panic(err) } +} + +fn do1() ! { + + mut p := os.new_process("/opt/homebrew/bin/python3") + p.set_work_folder("/tmp") + p.set_redirect_stdio() + p.use_stdio_ctl = true + p.use_pgroup = true + p.set_args(['-i','-q']) + p.run() + // p.set_args("") + // time.sleep(100 * time.millisecond) + println( "alive: ${p.is_alive()}") + assert p.is_alive() + + defer { + p.wait() + p.close() + } + + // for { + // println(1) + // println(p.stdout_slurp()) + // println(2) + // println(p.stderr_slurp()) + // println(3) + // } + mut counter:=0 + for { + counter+=1 + println(counter) + out:=p.pipe_read(.stdout) or {""} + if out.len>0{ + println("o") + println(out) + } + err:=p.pipe_read(.stderr) or {""} + if err.len>0{ + println("e") + println(err) + } + time.sleep(100 * time.millisecond) + if counter==2{ + p.stdin_write("print('something')\n\n\n") + // os.fd_close(p.stdio_fd[0]) + } + if counter==20{ + p.stdin_write("print('something else')\n\n\n") + // os.fd_close(p.stdio_fd[0]) + } + } +} + + + + + + +fn do2() ! { + + mut p := os.new_process("/opt/homebrew/bin/python3") + p.set_work_folder("/tmp") + p.set_redirect_stdio() + p.use_stdio_ctl = true + p.use_pgroup = true + p.run() + // p.set_args("") + // time.sleep(100 * time.millisecond) + println( "alive: ${p.is_alive()}") + assert p.is_alive() + + defer { + p.wait() + p.close() + } + + for { + fdi:=p.stdio_fd[0] + fdo:=p.stdio_fd[1] + fde:=p.stdio_fd[2] + println(1) + if os.fd_is_pending(fdo){ + println(1.1) + println(os.fd_slurp(fdo)) + } + println(2) + if os.fd_is_pending(fde){ + println(2.1) + println(os.fd_slurp(fde)) + } + println(3) + time.sleep(100 * time.millisecond) + } + mut counter:=0 + for { + counter+=1 + println(counter) + out:=p.pipe_read(.stdout) or {""} + if out.len>0{ + println("o") + println(out) + } + err:=p.pipe_read(.stderr) or {""} + if err.len>0{ + println("e") + println(err) + } + time.sleep(100 * time.millisecond) + if counter==2{ + p.stdin_write("print('something')\n\n\n") + os.fd_close(p.stdio_fd[0]) + } + if counter==20{ + p.stdin_write("print('something else')\n\n\n") + os.fd_close(p.stdio_fd[0]) + } + } +} + + + + + diff --git a/examples/osal/process/processmap.v b/examples/osal/process/processmap.v new file mode 100644 index 00000000..faff6148 --- /dev/null +++ b/examples/osal/process/processmap.v @@ -0,0 +1,12 @@ +module main + +import freeflowuniverse.herolib.osal + +fn do() ? { + mut pm := process.processmap_get()? + println(pm) +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/osal/rsync/rsync_example.v b/examples/osal/rsync/rsync_example.v new file mode 100644 index 00000000..f01965a9 --- /dev/null +++ b/examples/osal/rsync/rsync_example.v @@ -0,0 +1,50 @@ +module main + +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.builder +import os + +const myexamplepath = os.dir(@FILE) + '/../..' + +fn do1() ! { + tstdir := '/tmp/testsync' + // source string + // dest string + // delete bool //do we want to delete the destination + // ipaddr_src string //e.g. root@192.168.5.5:33 (can be without root@ or :port) + // ipaddr_dst string + // ignore []string //arguments to ignore e.g. ['*.pyc','*.bak'] + // ignore_default bool //if set will ignore a common set + // stdout bool + osal.rsync(source: myexamplepath, dest: tstdir, delete: true)! + cmd := osal.rsync_cmd(source: myexamplepath, dest: tstdir)! + println(cmd) + //"rsync -avz --no-perms --exclude='*.pyc' --exclude='*.bak' --exclude='*dSYM' /Users/despiegk1/code/github/freeflowuniverse/herolib/examples /tmp/testsync" +} + +fn do2() ! { + mut b := builder.new()! + mut n := b.node_new(ipaddr: 'root@195.192.213.2')! + tstdir := '/tmp/testsync' + n.exec('mkdir -p ${tstdir}')! + + ipaddr := 'root@195.192.213.2' + osal.rsync(source: myexamplepath, ipaddr_dst: ipaddr, dest: tstdir, delete: true)! + cmd := osal.rsync_cmd(source: myexamplepath, dest: tstdir)! + println(cmd) +} + +fn do3() ! { + ipaddr := 'root@195.192.213.2:22' + tstdir := '/tmp/testsync' + + osal.rsync(ipaddr_src: ipaddr, source: tstdir, dest: tstdir, delete: true)! + cmd := osal.rsync_cmd(source: tstdir, dest: tstdir)! + println(cmd) +} + +fn main() { + do1() or { panic(err) } + do2() or { panic(err) } + do3() or { panic(err) } +} diff --git a/examples/osal/sandbox/examples/.gitignore b/examples/osal/sandbox/examples/.gitignore new file mode 100644 index 00000000..02ad8162 --- /dev/null +++ b/examples/osal/sandbox/examples/.gitignore @@ -0,0 +1 @@ +sandbox_example diff --git a/examples/osal/sandbox/examples/sandbox_example.v b/examples/osal/sandbox/examples/sandbox_example.v new file mode 100644 index 00000000..09b3046c --- /dev/null +++ b/examples/osal/sandbox/examples/sandbox_example.v @@ -0,0 +1,26 @@ +module main + +import freeflowuniverse.herolib.osal.sandbox +import os + +fn do() ! { + sandbox.install()! // will also do an upgrade of the OS + + mut f := sandbox.new(path_images: '/var/sandbox/images')! + + // get 2 bootstraps to work from + f.debootstrap(imagename: 'debian', reset: false)! // if reset then will download again + f.debootstrap( + imagename: 'ubuntu22' + repository: 'http://de.archive.ubuntu.com/ubuntu' + release: 'jammy' + reset: false + )! + + // mut c := f.container_new(startcmd: ["ls", "/", "/proc"])! + // c.start()! +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/osal/sshagent/.gitignore b/examples/osal/sshagent/.gitignore new file mode 100644 index 00000000..d9670f3f --- /dev/null +++ b/examples/osal/sshagent/.gitignore @@ -0,0 +1 @@ +sshagent_example \ No newline at end of file diff --git a/examples/osal/sshagent/sshagent_example.v b/examples/osal/sshagent/sshagent_example.v new file mode 100644 index 00000000..ca7cbf94 --- /dev/null +++ b/examples/osal/sshagent/sshagent_example.v @@ -0,0 +1,28 @@ +module main + +import freeflowuniverse.herolib.osal.sshagent + +fn do1() ! { + mut agent:=sshagent.new()! + println(agent) + k:=agent.get(name:"kds") or {panic("notgound")} + println(k) + + mut k2:=agent.get(name:"books") or {panic("notgound")} + k2.load()! + println(k2.agent) + + println(agent) + + k2.forget()! + println(k2.agent) + + // println(agent) + +} + + + +fn main() { + do1() or { panic(err) } +} diff --git a/examples/osal/startup_manager.vsh b/examples/osal/startup_manager.vsh new file mode 100755 index 00000000..3b157e6c --- /dev/null +++ b/examples/osal/startup_manager.vsh @@ -0,0 +1,26 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.infra.zinit as zinitinstaller +import freeflowuniverse.herolib.sysadmin.startupmanager + +mut z:=zinitinstaller.get()! +z.destroy()! +z.install()! + +println("zinit installed") + +cmd:= '/usr/local/bin/zinit init' +name:= 'zinit' + +mut sm := startupmanager.get()! +println(sm.list()!) +sm.new( + name: name + cmd: cmd + start:false +)! + +println(sm.list()!) +assert sm.exists(name)! + +sm.delete(name)! \ No newline at end of file diff --git a/examples/osal/systemd.vsh b/examples/osal/systemd.vsh new file mode 100755 index 00000000..ff3110ed --- /dev/null +++ b/examples/osal/systemd.vsh @@ -0,0 +1,14 @@ +#!/usr/bin/env -S v -gc none -cg -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.osal.systemd + + +mut systemdfactory := systemd.new()! +// mut systemdprocess := systemdfactory.new( +// cmd: '/usr/local/bin/zinit init' +// name: 'zinit' +// description: 'a super easy to use startup manager.' +// )! +l:=systemd.process_list()! +println(l) +systemdfactory.destroy("zinit")! diff --git a/examples/osal/ufw.vsh b/examples/osal/ufw.vsh new file mode 100755 index 00000000..91fc16f5 --- /dev/null +++ b/examples/osal/ufw.vsh @@ -0,0 +1,41 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.osal.ufw + +ufw.enable()! +println(ufw.ufw_status()!) + +mut ruleset := ufw.new() + +// Allow HTTP traffic from a specific IPv4 address +ruleset.allow( + port: 80 + from: '192.168.1.100' +) + +// Allow HTTPS traffic from any IPv6 address +ruleset.allow( + port: 443 + ipv6: true +) + +// Deny SMTP traffic from a specific IPv4 subnet +ruleset.deny( + port: 25 + from: '10.0.0.0/24' +) + +// Deny FTP traffic from a specific IPv6 address +ruleset.deny( + port: 21 + from: '2001:db8::1' + udp: true + tcp: false + ipv6: true +) + +// Apply the ruleset +ufw.apply(ruleset) or { panic('Error applying ruleset: ${err}') } + +ufw.reset()! +ufw.enable()! \ No newline at end of file diff --git a/examples/osal/ufw_play.vsh b/examples/osal/ufw_play.vsh new file mode 100755 index 00000000..f1f98d10 --- /dev/null +++ b/examples/osal/ufw_play.vsh @@ -0,0 +1,40 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.osal.ufw +import freeflowuniverse.herolib.core.playbook + +heroscript := " + !!ufw.configure + active: true + ssh: true + reset: true + + !!ufw.add_rule + allow: true + port: 80 + from: 'any' + tcp: true + udp: false + ipv6: false + + !!ufw.add_rule + allow: false + port: 443 + from: '192.168.1.0/24' + tcp: true + udp: false + ipv6: false + + !!ufw.add_rule + allow: true + port: 53 + from: 'any' + tcp: true + udp: true + ipv6: true + " + +mut plbook := playbook.new(text: heroscript)! +rs:=ufw.play(mut plbook)! +println(rs) + diff --git a/examples/osal/zinit/rpc/service_1.yaml b/examples/osal/zinit/rpc/service_1.yaml new file mode 100644 index 00000000..856129a6 --- /dev/null +++ b/examples/osal/zinit/rpc/service_1.yaml @@ -0,0 +1 @@ +exec: "sleep 1m" \ No newline at end of file diff --git a/examples/osal/zinit/rpc/service_2.yaml b/examples/osal/zinit/rpc/service_2.yaml new file mode 100644 index 00000000..aba0a181 --- /dev/null +++ b/examples/osal/zinit/rpc/service_2.yaml @@ -0,0 +1,3 @@ +exec: "sleep 1m" +after: + - service_1 \ No newline at end of file diff --git a/examples/osal/zinit/rpc/zinit.v b/examples/osal/zinit/rpc/zinit.v new file mode 100644 index 00000000..2f684f3f --- /dev/null +++ b/examples/osal/zinit/rpc/zinit.v @@ -0,0 +1,68 @@ +module main + +import os +import time +import freeflowuniverse.herolib.osal.zinit + +fn main() { + do() or { panic(err) } +} + +fn do() ! { + start_zinit()! + client := zinit.new_rpc_client('herolib/osal/zinit/zinit/zinit.sock') + list_services(client)! + get_service_status(client, 'service_2')! + stop_service(client, 'service_2')! + forget_service(client, 'service_2')! + monitor_service(client, 'service_2')! + stop_service(client, 'service_2')! + start_service(client, 'service_2')! + kill_service(client, 'service_1', 'sigterm')! +} + +fn start_zinit() ! { + spawn os.execute('zinit -s examples/osal/zinit/zinit.sock init -c examples/osal/zinit') + time.sleep(time.second) +} + +fn list_services(client zinit.Client) ! { + mut ls := client.list()! + println('services watched by zinit: ${ls}\n\n') +} + +fn get_service_status(client zinit.Client, service_name string) ! { + time.sleep(time.millisecond * 100) + mut st := client.status(service_name)! + println('${service_name} status: ${st}\n\n') +} + +fn stop_service(client zinit.Client, service_name string) ! { + println('Stopping ${service_name}...') + client.stop(service_name)! + get_service_status(client, service_name)! +} + +fn forget_service(client zinit.Client, service_name string) ! { + println('Forgetting ${service_name}...') + client.forget(service_name)! + list_services(client)! +} + +fn monitor_service(client zinit.Client, service_name string) ! { + println('Monitoring service ${service_name}...') + client.monitor(service_name)! + get_service_status(client, service_name)! +} + +fn start_service(client zinit.Client, service_name string) ! { + println('Starting service ${service_name}...') + client.start(service_name)! + get_service_status(client, service_name)! +} + +fn kill_service(client zinit.Client, service_name string, sig string) ! { + println('Killing service ${service_name}...') + client.kill(service_name, sig)! + get_service_status(client, service_name)! +} diff --git a/examples/osal/zinit/simple/zinit.vsh b/examples/osal/zinit/simple/zinit.vsh new file mode 100644 index 00000000..1f3c8225 --- /dev/null +++ b/examples/osal/zinit/simple/zinit.vsh @@ -0,0 +1,24 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import os +import time +import freeflowuniverse.herolib.osal.zinit + +zinit.destroy()! +mut z := zinit.new()! + +// name string [required] +// cmd string [required] +// cmd_file bool //if we wanna force to run it as a file which is given to bash -c (not just a cmd in zinit) +// test string +// test_file bool +// after []string +// env map[string]string +// oneshot bool + +p := z.process_new( + name: 'test' + cmd: '/bin/bash' +)! + +println(p) diff --git a/examples/sync_do.sh b/examples/sync_do.sh new file mode 100755 index 00000000..16de2c93 --- /dev/null +++ b/examples/sync_do.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# SSH and rsync configuration +SSH_HOST="verse.tf" +SSH_USER="root" +SOURCE_DIR="${HOME}/code/github/freeflowuniverse/herolib/" +DEST_DIR="/root/code/github/freeflowuniverse/herolib/" +FINAL_DIR="/root/code/github/freeflowuniverse/herolib/examples/hero" + +# Check if the source directory exists, if not stop +if [ ! -d "$SOURCE_DIR" ]; then + echo "Source directory $SOURCE_DIR does not exist. Exiting." + exit 1 +fi + +# Perform rsync over SSH, ignoring .git directory +#--exclude '.git' --exclude '.venv' +rsync -avz --delete -e ssh "$SOURCE_DIR/" "$SSH_USER@$SSH_HOST:$DEST_DIR/" + +set -x + +# SSH into the remote machine and change to the specified directory +ssh -At root@verse.tf "tmux attach-session -t main || tmux new-session -s main -c ${FINAL_DIR}" diff --git a/examples/threefold/.gitignore b/examples/threefold/.gitignore new file mode 100644 index 00000000..38aecdeb --- /dev/null +++ b/examples/threefold/.gitignore @@ -0,0 +1 @@ +holochain_deployer diff --git a/examples/threefold/grid/README.md b/examples/threefold/grid/README.md new file mode 100644 index 00000000..a61e5025 --- /dev/null +++ b/examples/threefold/grid/README.md @@ -0,0 +1,90 @@ +# Installing `griddriver` + +To be able to run examples you need to install updated version of `griddriver`. + +## Install from crytallib installer + +Create some `griddriver_install.vsh` file containing following code: + +```vlang +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.tfgrid.griddriver as griddriverinstaller + +mut reset:=true +griddriverinstaller.install(reset:reset)! +``` + +Make script executable and run it + +```sh +chmod +x ./griddriver_install.vsh +./griddriver_install.vsh +``` + +## Install from repo + +Checkout the `griddriver` main branch +https://github.com/threefoldtech/web3gw/tree/development_integration + +Inside the web3gw directory, run: + +```sh +cd griddriver +./build.sh +``` + +# Run examples + +These example scripts demonstrate various functionalities and interactions with +the TFGrid using the Hero programming language. They provide a starting point +for developers to understand and build upon when working with the TFGrid API and +deploying resources on the grid. + +## Utils + +- `billing_hourly.vsh`: calculate the hourly billing for a specific contract + ID. +- `cancel_contract.vsh`: cancel a specific contract on the TFGrid. +- `cancel_contracts.vsh`: cancel multiple contracts on the TFGrid. +- `deploy_vm_high_level.vsh`: deploy a virtual machine (VM) on the TFGrid + using a high-level approach. +- `get_contracts.vsh`: retrieve a list of all active contracts associated with + the configured identity on the TFGrid. +- `list_gateways.vsh`: list all available gateways on the TFGrid. +- `tfgrid_config.vsh`: configure the connection settings for interacting with + the TFGrid. +- `zos_version.vsh`: check the version of the Zero-OS (ZOS) running on a + specific node. + +## Tests + +- `create_update_deployments.vsh`: create a deployment with various workloads + (network, disk, public IP, VM, logs, ZDB) and a gateway name proxy, deploy + it to a node, and update the deployment with the gateway name workload. +- `deploy_gw_fqdn.vsh`: deploy a gateway workload using a Fully Qualified + Domain Name (FQDN). +- `deploy_gw_name.vsh`: deploy a gateway workload using a name contract. It + creates a GatewayNameProxy workload, reserves the name on the grid using a + name contract, and deploys it to a specific node. +- `deploy_vm.vsh`: deploy a network (Znet) and a virtual machine (Zmachine). +- `deploy_zdb.vsh`: deploy a ZDB (Zero-DB) workload on a specific node. +- `holochain_vm.vsh`: set up a Holochain development environment on the + ThreeFold Grid without manual configuration. The script is related to + Holochain because it specifically deploys a Holochain development + environment on the ThreeFold Grid. The Flist URL used in the virtual machine + workload points to a pre-built Holochain development environment image. + Usage: + +```sh +./holochain_vm.vsh --mnemonic "your_mnemonic_phrase" --ssh_key "your_public_ssh_key" [--network main|test|qa|dev] [--code_server_pass "your_password"] [--cpu 4] [--ram 8] [--disk 30] [--public_ip] +``` + +- `vm_with_gw_name.vsh`: deploy a VM workload along with a gateway using a + name contract. It finds a node matching the VM capacity requirements, + creates a network, a VM, and a gateway workload pointing to the VM. It then + deploys the VM and gateway workloads to their respective nodes. Usage: + +```sh +./vm_with_gw_name.vsh --mnemonic "your_mnemonic_phrase" --ssh_key "your_public_ssh_key" [--network main|test|qa|dev] [--cpu 4] [--ram 4] [--disk 5] [--public_ip] +``` diff --git a/examples/threefold/grid/deploy/create_update_deployments.vsh b/examples/threefold/grid/deploy/create_update_deployments.vsh new file mode 100755 index 00000000..4d3f5cbc --- /dev/null +++ b/examples/threefold/grid/deploy/create_update_deployments.vsh @@ -0,0 +1,150 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid.models +import freeflowuniverse.herolib.threefold.grid as tfgrid +import json +import log + +const pubkey = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDTwULSsUubOq3VPWL6cdrDvexDmjfznGydFPyaNcn7gAL9lRxwFbCDPMj7MbhNSpxxHV2+/iJPQOTVJu4oc1N7bPP3gBCnF51rPrhTpGCt5pBbTzeyNweanhedkKDsCO2mIEh/92Od5Hg512dX4j7Zw6ipRWYSaepapfyoRnNSriW/s3DH/uewezVtL5EuypMdfNngV/u2KZYWoeiwhrY/yEUykQVUwDysW/xUJNP5o+KSTAvNSJatr3FbuCFuCjBSvageOLHePTeUwu6qjqe+Xs4piF1ByO/6cOJ8bt5Vcx0bAtI8/MPApplUU/JWevsPNApvnA/ntffI+u8DCwgP' + +fn test_create_and_update_deployment() ! { + mut logger := &log.Log{} + logger.set_level(.debug) + mnemonics := tfgrid.get_mnemonics()! + mut deployer := tfgrid.new_deployer(mnemonics, .dev, mut logger)! + node_privkey := deployer.client.generate_wg_priv_key()! + user_privkey := deployer.client.generate_wg_priv_key()! + twin_id := deployer.client.get_user_twin()! + println('your wireguard privatekey is ${user_privkey[0]}') + mut network := models.Znet{ + ip_range: '10.1.0.0/16' + subnet: '10.1.1.0/24' + wireguard_private_key: node_privkey[0] // node private key + wireguard_listen_port: 3012 + peers: [ + models.Peer{ + subnet: '10.1.2.0/24' + wireguard_public_key: user_privkey[1] // user public key + allowed_ips: ['10.1.2.0/24', '100.64.1.2/32'] + }, + ] + } + mut znet_workload := models.Workload{ + version: 0 + name: 'networkaa' + type_: models.workload_types.network + data: json.encode_pretty(network) + description: 'test network2' + } + + disk_name := 'mydisk' + zmount := models.Zmount{ + size: 2 * 1024 * 1024 * 1024 + } + zmount_workload := zmount.to_workload(name: disk_name) + + mount := models.Mount{ + name: disk_name + mountpoint: '/disk1' + } + + public_ip_name := 'mypubip' + ip := models.PublicIP{ + v4: true + } + ip_workload := ip.to_workload(name: public_ip_name) + + zmachine := models.Zmachine{ + flist: 'https://hub.grid.tf/tf-official-apps/base:latest.flist' + entrypoint: '/sbin/zinit init' + network: models.ZmachineNetwork{ + public_ip: public_ip_name + interfaces: [ + models.ZNetworkInterface{ + network: 'networkaa' + ip: '10.1.1.3' + }, + ] + planetary: true + } + compute_capacity: models.ComputeCapacity{ + cpu: 1 + memory: i64(1024) * 1024 * 1024 * 2 + } + env: { + 'SSH_KEY': pubkey + } + mounts: [mount] + } + + mut zmachine_workload := models.Workload{ + version: 0 + name: 'vm2' + type_: models.workload_types.zmachine + data: json.encode(zmachine) + description: 'zmachine test' + } + + zlogs := models.ZLogs{ + zmachine: 'vm2' + output: 'wss://example_ip.com:9000' + } + zlogs_workload := zlogs.to_workload(name: 'myzlogswl') + + zdb := models.Zdb{ + size: 2 * 1024 * 1024 + mode: 'seq' + } + zdb_workload := zdb.to_workload(name: 'myzdb') + + mut deployment := models.Deployment{ + version: 0 + twin_id: twin_id + description: 'zm kjasdf1nafvbeaf1234t21' + workloads: [znet_workload, zmount_workload, zmachine_workload, zlogs_workload, zdb_workload, + ip_workload] + signature_requirement: models.SignatureRequirement{ + weight_required: 1 + requests: [ + models.SignatureRequest{ + twin_id: twin_id + weight: 1 + }, + ] + } + } + + deployment.add_metadata('myproject', 'hamada') + node_id := u32(14) + solution_provider := u64(0) + + contract_id := deployer.deploy(node_id, mut deployment, deployment.metadata, solution_provider)! + deployer.logger.info('created contract id ${contract_id}') + + res_deployment := deployer.get_deployment(contract_id, node_id)! + + mut zmachine_planetary_ip := '' + for wl in res_deployment.workloads { + if wl.name == zmachine_workload.name { + res := json.decode(models.ZmachineResult, wl.result.data)! + zmachine_planetary_ip = res.planetary_ip + break + } + } + + gw_name := models.GatewayNameProxy{ + name: 'mygwname1' + backends: ['http://[${zmachine_planetary_ip}]:9000'] + } + gw_name_wl := gw_name.to_workload(name: 'mygwname1') + + name_contract_id := deployer.client.create_name_contract('mygwname1')! + deployer.logger.info('name contract id: ${name_contract_id}') + + deployment.workloads << gw_name_wl + deployer.update_deployment(node_id, mut deployment, deployment.metadata)! +} + +fn main() { + test_create_and_update_deployment() or { println('error happened: ${err}') } +} diff --git a/examples/threefold/grid/deploy/deploy_gw_fqdn.vsh b/examples/threefold/grid/deploy/deploy_gw_fqdn.vsh new file mode 100755 index 00000000..2f5008b9 --- /dev/null +++ b/examples/threefold/grid/deploy/deploy_gw_fqdn.vsh @@ -0,0 +1,51 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid as tfgrid +import freeflowuniverse.herolib.threefold.grid.models +import log + +fn main() { + mut logger := &log.Log{} + logger.set_level(.debug) + + mnemonics := tfgrid.get_mnemonics() or { + logger.error(err.str()) + exit(1) + } + chain_network := tfgrid.ChainNetwork.dev // User your desired network + mut deployer := tfgrid.new_deployer(mnemonics, chain_network, mut logger)! + + gw := models.GatewayFQDNProxy{ + tls_passthrough: false + backends: ['http://1.1.1.1:9000'] + fqdn: 'domaind.gridtesting.xyz' + } + wl := gw.to_workload(name: 'mywlname') + node_id := u32(14) + logger.info('trying to get node ${node_id} public configuration') + deployer.get_node_pub_config(node_id) or { + logger.error('please select another node: ${err}') + exit(1) + } + logger.info('preparing the deployment..') + signature_requirement := models.SignatureRequirement{ + weight_required: 1 + requests: [ + models.SignatureRequest{ + twin_id: deployer.twin_id + weight: 1 + }, + ] + } + mut deployment := models.new_deployment( + twin_id: deployer.twin_id + workloads: [wl] + signature_requirement: signature_requirement + ) + + node_contract_id := deployer.deploy(node_id, mut deployment, '', 0) or { + logger.error(err.str()) + exit(1) + } + logger.info('node contract created with id ${node_contract_id}') +} diff --git a/examples/threefold/grid/deploy/deploy_gw_name.vsh b/examples/threefold/grid/deploy/deploy_gw_name.vsh new file mode 100755 index 00000000..7530741b --- /dev/null +++ b/examples/threefold/grid/deploy/deploy_gw_name.vsh @@ -0,0 +1,48 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid as tfgrid +import freeflowuniverse.herolib.threefold.grid.models +import log + +fn main() { + mut logger := &log.Log{} + logger.set_level(.debug) + + mnemonics := tfgrid.get_mnemonics() or { + logger.error(err.str()) + exit(1) + } + chain_network := tfgrid.ChainNetwork.dev // User your desired network + mut deployer := tfgrid.new_deployer(mnemonics, chain_network, mut logger)! + + gw := models.GatewayNameProxy{ + tls_passthrough: false + backends: ['http://1.1.1.1'] + name: 'hamada_gw' + } + + wl := gw.to_workload(name: 'hamada_gw') + + name_contract_id := deployer.client.create_name_contract(wl.name)! + logger.info('name contract ${wl.name} created with id ${name_contract_id}') + + signature_requirement := models.SignatureRequirement{ + weight_required: 1 + requests: [ + models.SignatureRequest{ + twin_id: deployer.twin_id + weight: 1 + }, + ] + } + + mut deployment := models.new_deployment( + twin_id: deployer.twin_id + workloads: [wl] + signature_requirement: signature_requirement + ) + + node_id := u32(14) + node_contract_id := deployer.deploy(node_id, mut deployment, '', 0)! + logger.info('node contract created with id ${node_contract_id}') +} diff --git a/examples/threefold/grid/deploy/deploy_vm.vsh b/examples/threefold/grid/deploy/deploy_vm.vsh new file mode 100755 index 00000000..7503c26a --- /dev/null +++ b/examples/threefold/grid/deploy/deploy_vm.vsh @@ -0,0 +1,98 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid.models +import freeflowuniverse.herolib.threefold.grid as tfgrid +import json +import log +import os + +fn main() { + mut logger := &log.Log{} + logger.set_level(.debug) + mnemonics := os.getenv('TFGRID_MNEMONIC') + chain_network := tfgrid.ChainNetwork.dev // User your desired network + mut deployer := tfgrid.new_deployer(mnemonics, chain_network, mut logger)! + + node_id := u32(14) + network_name := 'network1' + wg_port := deployer.assign_wg_port(node_id)! + mut network := models.Znet{ + ip_range: '10.1.0.0/16' + subnet: '10.1.1.0/24' + wireguard_private_key: 'GDU+cjKrHNJS9fodzjFDzNFl5su3kJXTZ3ipPgUjOUE=' + wireguard_listen_port: wg_port + peers: [ + models.Peer{ + subnet: '10.1.2.0/24' + wireguard_public_key: '4KTvZS2KPWYfMr+GbiUUly0ANVg8jBC7xP9Bl79Z8zM=' + allowed_ips: ['10.1.2.0/24', '100.64.1.2/32'] + }, + ] + } + mut znet_workload := network.to_workload(name: network_name, description: 'test_network1') + + zmachine := models.Zmachine{ + flist: 'https://hub.grid.tf/tf-official-apps/threefoldtech-ubuntu-22.04.flist' + network: models.ZmachineNetwork{ + public_ip: '' + interfaces: [ + models.ZNetworkInterface{ + network: network_name + ip: '10.1.1.3' + }, + ] + planetary: true + } + entrypoint: '/sbin/zinit init' + compute_capacity: models.ComputeCapacity{ + cpu: 1 + memory: i64(1024) * 1024 * 1024 * 2 + } + env: { + 'SSH_KEY': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDTwULSsUubOq3VPWL6cdrDvexDmjfznGydFPyaNcn7gAL9lRxwFbCDPMj7MbhNSpxxHV2+/iJPQOTVJu4oc1N7bPP3gBCnF51rPrhTpGCt5pBbTzeyNweanhedkKDsCO2mIEh/92Od5Hg512dX4j7Zw6ipRWYSaepapfyoRnNSriW/s3DH/uewezVtL5EuypMdfNngV/u2KZYWoeiwhrY/yEUykQVUwDysW/xUJNP5o+KSTAvNSJatr3FbuCFuCjBSvageOLHePTeUwu6qjqe+Xs4piF1ByO/6cOJ8bt5Vcx0bAtI8/MPApplUU/JWevsPNApvnA/ntffI+u8DCwgP' + } + } + mut zmachine_workload := zmachine.to_workload(name: 'vm2', description: 'zmachine_test') + + signature_requirement := models.SignatureRequirement{ + weight_required: 1 + requests: [ + models.SignatureRequest{ + twin_id: deployer.twin_id + weight: 1 + }, + ] + } + + mut deployment := models.new_deployment( + twin_id: deployer.twin_id + description: 'test deployment' + workloads: [znet_workload, zmachine_workload] + signature_requirement: signature_requirement + ) + deployment.add_metadata('vm', 'SimpleVM') + + contract_id := deployer.deploy(node_id, mut deployment, deployment.metadata, 0) or { + logger.error('failed to deploy deployment: ${err}') + exit(1) + } + logger.info('deployment contract id: ${contract_id}') + dl := deployer.get_deployment(contract_id, node_id) or { + logger.error('failed to get deployment data: ${err}') + exit(1) + } + + machine_res := get_machine_result(dl)! + logger.info('zmachine result: ${machine_res}') +} + +fn get_machine_result(dl models.Deployment) !models.ZmachineResult { + for _, w in dl.workloads { + if w.type_ == models.workload_types.zmachine { + res := json.decode(models.ZmachineResult, w.result.data)! + return res + } + } + + return error('failed to get zmachine workload') +} diff --git a/examples/threefold/grid/deploy/deploy_vm_high_level.vsh b/examples/threefold/grid/deploy/deploy_vm_high_level.vsh new file mode 100755 index 00000000..0b42d1eb --- /dev/null +++ b/examples/threefold/grid/deploy/deploy_vm_high_level.vsh @@ -0,0 +1,30 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid.models +import freeflowuniverse.herolib.threefold.grid as tfgrid +import log +import os + +fn test_deploy_vm_hight_level(node_id u32) ! { + mnemonics := tfgrid.get_mnemonics()! + chain_network := tfgrid.ChainNetwork.dev // User your desired network + + mut logger := &log.Log{} + logger.set_level(.debug) + + mut deployer := tfgrid.new_deployer(mnemonics, chain_network, mut logger)! + + vm := models.VM{ + name: 'vm1' + env_vars: { + 'SSH_KEY': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDTwULSsUubOq3VPWL6cdrDvexDmjfznGydFPyaNcn7gAL9lRxwFbCDPMj7MbhNSpxxHV2+/iJPQOTVJu4oc1N7bPP3gBCnF51rPrhTpGCt5pBbTzeyNweanhedkKDsCO2mIEh/92Od5Hg512dX4j7Zw6ipRWYSaepapfyoRnNSriW/s3DH/uewezVtL5EuypMdfNngV/u2KZYWoeiwhrY/yEUykQVUwDysW/xUJNP5o+KSTAvNSJatr3FbuCFuCjBSvageOLHePTeUwu6qjqe+Xs4piF1ByO/6cOJ8bt5Vcx0bAtI8/MPApplUU/JWevsPNApvnA/ntffI+u8DCwgP' + } + } + res := deployer.client.deploy_single_vm(node_id, 'myproject', vm, deployer.env)! + + deployer.logger.info('${res}') +} + +fn main() { + test_deploy_vm_hight_level(u32(14)) or { println('error happened: ${err}') } +} diff --git a/examples/threefold/grid/deploy/deploy_zdb.vsh b/examples/threefold/grid/deploy/deploy_zdb.vsh new file mode 100755 index 00000000..c816fc05 --- /dev/null +++ b/examples/threefold/grid/deploy/deploy_zdb.vsh @@ -0,0 +1,45 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid.models +import freeflowuniverse.herolib.threefold.grid as tfgrid +import log + +fn main() { + mut logger := &log.Log{} + logger.set_level(.debug) + + mnemonics := tfgrid.get_mnemonics() or { + logger.error(err.str()) + exit(1) + } + chain_network := tfgrid.ChainNetwork.dev // Use your desired network + mut deployer := tfgrid.new_deployer(mnemonics, chain_network, mut logger)! + + zdb := models.Zdb{ + size: u64(2) * 1024 * 1024 + mode: 'user' + password: 'pass' + } + + wl := zdb.to_workload(name: 'mywlname') + + signature_requirement := models.SignatureRequirement{ + weight_required: 1 + requests: [ + models.SignatureRequest{ + twin_id: deployer.twin_id + weight: 1 + }, + ] + } + + mut deployment := models.new_deployment( + twin_id: deployer.twin_id + workloads: [wl] + signature_requirement: signature_requirement + ) + + node_id := u32(14) + node_contract_id := deployer.deploy(node_id, mut deployment, '', 0)! + logger.info('node contract created with id ${node_contract_id}') +} diff --git a/examples/threefold/grid/deploy/holochain_vm.vsh b/examples/threefold/grid/deploy/holochain_vm.vsh new file mode 100755 index 00000000..17798262 --- /dev/null +++ b/examples/threefold/grid/deploy/holochain_vm.vsh @@ -0,0 +1,185 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid.models +import freeflowuniverse.herolib.threefold.grid as tfgrid +import freeflowuniverse.herolib.threefold.gridproxy +import flag +import rand +import json +import log +import os + +fn main() { + mut fp := flag.new_flag_parser(os.args) + fp.application('Holochain dev tool') + fp.version('v0.0.1') + fp.skip_executable() + + mnemonics := fp.string_opt('mnemonic', `m`, 'Your Mnemonic phrase')! + chain_network := fp.string('network', `n`, 'main', 'Your desired chain network (main, test, qa, dev). Defaults to main') + ssh_key := fp.string_opt('ssh_key', `s`, 'Your public ssh key')! + code_server_pass := fp.string('code_server_pass', `p`, 'password', 'Machine code server password. This will be set as a password for the code server on the deployed machine. Defaults to password') + cpu := fp.int('cpu', `c`, 4, 'Machine CPU provisioning. Defaults to 4') + memory := fp.int('ram', `r`, 8, 'Machine memory provisioning in GB. Defaults to 8') + disk := fp.int('disk', `d`, 30, 'Machine Disk space provisioning in GB. Defaults to 30') + public_ip := fp.bool('public_ip', `i`, false, 'True to allow public ip v4') + + mut logger := &log.Log{} + logger.set_level(.debug) + + chain_net_enum := get_chain_network(chain_network)! + mut deployer := tfgrid.new_deployer(mnemonics, chain_net_enum, mut logger)! + + mut workloads := []models.Workload{} + node_id := get_node_id(chain_net_enum, memory, disk, cpu, public_ip)! + // node_id := u32(150) + logger.info('deploying on node: ${node_id}') + + network_name := 'net_${rand.string(5).to_lower()}' // autocreate a network + wg_port := deployer.assign_wg_port(node_id)! + mut network := models.Znet{ + ip_range: '10.1.0.0/16' // auto-assign + subnet: '10.1.1.0/24' // auto-assign + wireguard_private_key: 'GDU+cjKrHNJS9fodzjFDzNFl5su3kJXTZ3ipPgUjOUE=' // autocreate + wireguard_listen_port: wg_port + mycelium: models.Mycelium{ + hex_key: rand.string(32).bytes().hex() + } + + } + + workloads << network.to_workload(name: network_name, description: 'test_network1') + + mut public_ip_name := '' + if public_ip{ + public_ip_name = rand.string(5).to_lower() + workloads << models.PublicIP{ + v4: true + }.to_workload(name: public_ip_name) + } + + zmachine := models.Zmachine{ + flist: 'https://hub.grid.tf/mariobassem1.3bot/threefolddev-holochain-latest.flist' // from user or default to ubuntu + network: models.ZmachineNetwork{ + interfaces: [ + models.ZNetworkInterface{ + network: network_name + ip: '10.1.1.3' + }, + ] + public_ip: public_ip_name + planetary: true + mycelium: models.MyceliumIP{ + network: network_name + hex_seed: rand.string(6).bytes().hex() + } + } + entrypoint: '/sbin/zinit init' // from user or default + compute_capacity: models.ComputeCapacity{ + cpu: u8(cpu) + memory: i64(memory) * 1024 * 1024 * 1024 + } + size: u64(disk) * 1024 * 1024 * 1024 + env: { + 'SSH_KEY': ssh_key + 'CODE_SERVER_PASSWORD': code_server_pass + } + } + + workloads << zmachine.to_workload( + name: 'vm_${rand.string(5).to_lower()}' + description: 'zmachine_test' + ) + + signature_requirement := models.SignatureRequirement{ + weight_required: 1 + requests: [ + models.SignatureRequest{ + twin_id: deployer.twin_id + weight: 1 + }, + ] + } + + mut deployment := models.new_deployment( + twin_id: deployer.twin_id + description: 'holochain deployment' + workloads: workloads + signature_requirement: signature_requirement + ) + deployment.add_metadata('vm', 'SimpleVM') + + contract_id := deployer.deploy(node_id, mut deployment, deployment.metadata, 0) or { + logger.error('failed to deploy deployment: ${err}') + exit(1) + } + logger.info('deployment contract id: ${contract_id}') + dl := deployer.get_deployment(contract_id, node_id) or { + logger.error('failed to get deployment data: ${err}') + exit(1) + } + + // logger.info('deployment:\n${dl}') + machine_res := get_machine_result(dl)! + logger.info('zmachine result: ${machine_res}') +} + +fn get_machine_result(dl models.Deployment) !models.ZmachineResult { + for _, w in dl.workloads { + if w.type_ == models.workload_types.zmachine { + res := json.decode(models.ZmachineResult, w.result.data)! + return res + } + } + + return error('failed to get zmachine workload') +} + +fn get_chain_network(network string) !tfgrid.ChainNetwork { + chain_net_enum := match network { + 'dev' { tfgrid.ChainNetwork.dev } + 'qa' { tfgrid.ChainNetwork.qa } + 'test' { tfgrid.ChainNetwork.test } + 'main' { tfgrid.ChainNetwork.main } + else { return error('invalid chain newtork ${network}. must be one of (dev, qa, test, main)') } + } + + return chain_net_enum +} + +fn get_node_id(network tfgrid.ChainNetwork, memory int, disk int, cpu int, public_ip bool) !u32{ + gp_net := match network { + .dev { gridproxy.TFGridNet.dev } + .qa { gridproxy.TFGridNet.qa } + .test { gridproxy.TFGridNet.test } + .main { gridproxy.TFGridNet.main } + } + + mut gridproxy_client := gridproxy.get(gp_net, false)! + mut free_ips := u64(0) + if public_ip{ + free_ips = 1 + } + + mut node_it := gridproxy_client.get_nodes_has_resources( + free_mru_gb: u64(memory) + free_sru_gb: u64(disk) + free_cpu: u64(cpu) + free_ips: free_ips + ) + nodes := node_it.next() + mut node_id := u32(0) // get from user or use gridproxy to get nodeid + if nodes_list := nodes { + node_id = u32(nodes_list[0].node_id) + } else { + return error('cannot find a suitable node matching your specs') + } + + return node_id +} + +/* + gridproxy call to assign node - done + generate private key for wireguard + add option to add public ip +*/ diff --git a/examples/threefold/grid/deploy/vm_with_gw_name.vsh b/examples/threefold/grid/deploy/vm_with_gw_name.vsh new file mode 100755 index 00000000..9893ed14 --- /dev/null +++ b/examples/threefold/grid/deploy/vm_with_gw_name.vsh @@ -0,0 +1,218 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid.models +import freeflowuniverse.herolib.threefold.grid as tfgrid +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.threefold.gridproxy.model {NodeFilter} +import rand +import log +import os +import flag +import json + +fn get_machine_result(dl models.Deployment) !models.ZmachineResult { + for _, w in dl.workloads { + if w.type_ == models.workload_types.zmachine { + res := json.decode(models.ZmachineResult, w.result.data)! + return res + } + } + + return error('failed to get zmachine workload') +} + +fn get_gateway_name_result(dl models.Deployment) !models.GatewayProxyResult { + for _, w in dl.workloads { + if w.type_ == models.workload_types.gateway_name { + res := json.decode(models.GatewayProxyResult, w.result.data)! + return res + } + } + + return error('failed to get gateway_name workload') +} + +fn get_chain_network(network string) !tfgrid.ChainNetwork { + chain_net_enum := match network { + 'dev' { tfgrid.ChainNetwork.dev } + 'qa' { tfgrid.ChainNetwork.qa } + 'test' { tfgrid.ChainNetwork.test } + 'main' { tfgrid.ChainNetwork.main } + else { return error('invalid chain newtork ${network}. must be one of (dev, qa, test, main)') } + } + + return chain_net_enum +} + +fn get_node_id(network tfgrid.ChainNetwork, memory int, disk int, cpu int, public_ip bool, has_domain bool, available_for u64) !u32{ + gp_net := match network { + .dev { gridproxy.TFGridNet.dev } + .qa { gridproxy.TFGridNet.qa } + .test { gridproxy.TFGridNet.test } + .main { gridproxy.TFGridNet.main } + } + + mut gridproxy_client := gridproxy.get(gp_net, false)! + mut free_ips := u64(0) + if public_ip{ + free_ips = 1 + } + + mut filter_ := NodeFilter{ + free_ips: free_ips + free_mru: u64(memory) * (1204 * 1204 * 1204) + free_sru: u64(disk) * (1204 * 1204 * 1204) + total_cru: u64(cpu) + domain: has_domain + available_for: available_for + status: 'up' + randomize: true + size: u64(1) + } + + nodes := gridproxy_client.get_nodes(filter_)! + if nodes.len != 1{ + return error('cannot find a suitable node matching your specs') + } + + return u32(nodes[0].node_id) +} + +mut fp := flag.new_flag_parser(os.args) +fp.application('VM with gateway deployer tool') +fp.version('v0.0.1') +fp.skip_executable() + +mnemonics := fp.string_opt('mnemonic', `m`, 'Your Mnemonic phrase')! +chain_network := fp.string('network', `n`, 'main', 'Your desired chain network (main, test, qa, dev). Defaults to main') +ssh_key := fp.string_opt('ssh_key', `s`, 'Your public ssh key')! +cpu := fp.int('cpu', `c`, 4, 'Machine CPU provisioning. Defaults to 4') +memory := fp.int('ram', `r`, 4, 'Machine memory provisioning in GB. Defaults to 4') +disk := fp.int('disk', `d`, 5, 'Machine Disk space provisioning in GB. Defaults to 5') +public_ip := fp.bool('public_ip', `i`, false, 'True to allow public ip v4') + +mut logger := &log.Log{} +logger.set_level(.debug) + +chain_net_enum := get_chain_network(chain_network)! +mut deployer := tfgrid.new_deployer(mnemonics, chain_net_enum, mut logger)! + +mut workloads := []models.Workload{} +node_id := get_node_id(chain_net_enum, memory, disk, cpu, public_ip, false, deployer.twin_id)! +// node_id := u32(150) +logger.info('deploying on node: ${node_id}') + +network_name := 'net_${rand.string(5).to_lower()}' // autocreate a network +wg_port := deployer.assign_wg_port(node_id)! +mut network := models.Znet{ + ip_range: '10.1.0.0/16' // auto-assign + subnet: '10.1.1.0/24' // auto-assign + wireguard_private_key: 'GDU+cjKrHNJS9fodzjFDzNFl5su3kJXTZ3ipPgUjOUE=' // autocreate + wireguard_listen_port: wg_port + // mycelium: models.Mycelium{ + // hex_key: rand.string(32).bytes().hex() + // } +} + +workloads << network.to_workload(name: network_name, description: 'test_network1') + +mut public_ip_name := '' +if public_ip{ + public_ip_name = rand.string(5).to_lower() + workloads << models.PublicIP{ + v4: true + }.to_workload(name: public_ip_name) +} + +zmachine := models.Zmachine{ + flist: 'https://hub.grid.tf/tf-official-apps/base:latest.flist' + network: models.ZmachineNetwork{ + interfaces: [ + models.ZNetworkInterface{ + network: network_name + ip: '10.1.1.3' + }, + ] + public_ip: public_ip_name + planetary: true + // mycelium: models.MyceliumIP{ + // network: network_name + // hex_seed: rand.string(6).bytes().hex() + // } + } + entrypoint: '/sbin/zinit init' // from user or default + compute_capacity: models.ComputeCapacity{ + cpu: u8(cpu) + memory: i64(memory) * 1024 * 1024 * 1024 + } + size: u64(disk) * 1024 * 1024 * 1024 + env: { + 'SSH_KEY': ssh_key + } +} + +workloads << zmachine.to_workload( + name: 'vm_${rand.string(5).to_lower()}' + description: 'zmachine_test' +) + +signature_requirement := models.SignatureRequirement{ + weight_required: 1 + requests: [ + models.SignatureRequest{ + twin_id: deployer.twin_id + weight: 1 + }, + ] +} + +mut deployment := models.new_deployment( + twin_id: deployer.twin_id + description: 'vm with gateway' + workloads: workloads + signature_requirement: signature_requirement +) +deployment.add_metadata('vm', 'SimpleVM') + +contract_id := deployer.deploy(node_id, mut deployment, deployment.metadata, 0) or { + logger.error('failed to deploy deployment: ${err}') + exit(1) +} +logger.info('deployment contract id: ${contract_id}') +dl := deployer.get_deployment(contract_id, node_id) or { + logger.error('failed to get deployment data: ${err}') + exit(1) +} + +machine_res := get_machine_result(dl)! +logger.info('zmachine result: ${machine_res}') + +gw_name := rand.string(5).to_lower() +gw := models.GatewayNameProxy{ + tls_passthrough: false + backends: ['http://[${machine_res.planetary_ip}]:9000'] + name: gw_name +} + +gw_workload := gw.to_workload(name: gw_name) + +name_contract_id := deployer.client.create_name_contract(gw_name)! +logger.info('name contract ${gw_workload.name} created with id ${name_contract_id}') + +mut gw_deployment := models.new_deployment( + twin_id: deployer.twin_id + workloads: [gw_workload] + signature_requirement: signature_requirement +) + +gw_node_id := get_node_id(chain_net_enum, 0, 0, 0, false, true, deployer.twin_id)! +gw_node_contract_id := deployer.deploy(gw_node_id, mut gw_deployment, '', 0)! +logger.info('gateway node contract created with id ${gw_node_contract_id}') + +gateway_dl := deployer.get_deployment(gw_node_contract_id, gw_node_id) or { + logger.error('failed to get deployment data: ${err}') + exit(1) +} + +gw_res := get_gateway_name_result(gateway_dl)! +logger.info('gateway: ${gw_res}') \ No newline at end of file diff --git a/examples/threefold/grid/deployment_state.vsh b/examples/threefold/grid/deployment_state.vsh new file mode 100644 index 00000000..cb7a0b72 --- /dev/null +++ b/examples/threefold/grid/deployment_state.vsh @@ -0,0 +1,39 @@ + + +struct DeploymentStateDB{ + secret ... //to encrypt symmetric + //... + + +} + + +struct DeploymentState{ + name ... + vms []VMDeployed + zdbs []ZDBDeployed + ... + +} + +pub fn (db DeploymentStateDB) set(deployment_name string, key string, val string)! { + //store e.g. \n separated list of all keys per deployment_name + //encrypt + +} + +pub fn (db DeploymentStateDB) get(deployment_name string, key string)!string { + +} + +pub fn (db DeploymentStateDB) delete(deployment_name string, key string)! { + +} + +pub fn (db DeploymentStateDB) keys(deployment_name string)![]string { + +} + +pub fn (db DeploymentStateDB) load(deployment_name string)!DeploymentState { + +} \ No newline at end of file diff --git a/examples/threefold/grid/utils/cancel_contract.vsh b/examples/threefold/grid/utils/cancel_contract.vsh new file mode 100755 index 00000000..72a8acdf --- /dev/null +++ b/examples/threefold/grid/utils/cancel_contract.vsh @@ -0,0 +1,18 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid as tfgrid +import log + +fn test_cancel_contract(contract_id u64) ! { + mut logger := &log.Log{} + logger.set_level(.debug) + mnemonics := tfgrid.get_mnemonics()! + chain_network := tfgrid.ChainNetwork.dev // User your desired network + mut deployer := tfgrid.new_deployer(mnemonics, chain_network, mut logger)! + deployer.client.cancel_contract(contract_id)! + deployer.logger.info('contract ${contract_id} is canceled') +} + +fn main() { + test_cancel_contract(u64(119497)) or { println('error happened: ${err}') } +} diff --git a/examples/threefold/grid/utils/cancel_contracts.vsh b/examples/threefold/grid/utils/cancel_contracts.vsh new file mode 100755 index 00000000..94ac5222 --- /dev/null +++ b/examples/threefold/grid/utils/cancel_contracts.vsh @@ -0,0 +1,19 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid as tfgrid +import log + +fn test_cancel_contracts(contracts_ids []u64) ! { + mut logger := &log.Log{} + logger.set_level(.debug) + mnemonics := tfgrid.get_mnemonics()! + mut deployer := tfgrid.new_deployer(mnemonics, .dev, mut logger)! + for cont_id in contracts_ids { + deployer.client.cancel_contract(cont_id)! + deployer.logger.info('contract ${cont_id} is canceled') + } +} + +fn main() { + test_cancel_contracts([u64(119493), u64(119492)]) or { println('error happened: ${err}') } +} diff --git a/examples/threefold/grid/utils/tfgrid_config.vsh b/examples/threefold/grid/utils/tfgrid_config.vsh new file mode 100755 index 00000000..93e24893 --- /dev/null +++ b/examples/threefold/grid/utils/tfgrid_config.vsh @@ -0,0 +1,37 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid as tfgrid + +mut cl := tfgrid.get("my_config")! +mut cfg := cl.config()! + +println(cl.instance) +cfg = cl.config()! +println(cfg) + +if cfg.mnemonics == "" { + // will ask questions if not filled in yet + cl.config_interactive()! +} + +println(cl.instance) +cfg = cl.config()! +println(cfg) + +// cl.instance = 'new_name' +cfg.mnemonics = '' +cfg.network = 'qa' +cl.config_save()! + +println(cl.instance) +cfg = cl.config()! +println(cfg) + +cl = tfgrid.get("empty_config")! + +println(cl.instance) +cfg = cl.config()! +println(cfg) + +// TO CONFIGURE NEW +// cl.config_delete()! \ No newline at end of file diff --git a/examples/threefold/grid/utils/zos_version.vsh b/examples/threefold/grid/utils/zos_version.vsh new file mode 100755 index 00000000..3eb9e8e9 --- /dev/null +++ b/examples/threefold/grid/utils/zos_version.vsh @@ -0,0 +1,21 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid as tfgrid +import freeflowuniverse.herolib.threefold.griddriver { Client } +import freeflowuniverse.herolib.ui.console +import log + +fn test_get_zos_version(node_id u32) ! { + mut logger := &log.Log{} + logger.set_level(.debug) + mnemonics := tfgrid.get_mnemonics()! + chain_network := tfgrid.ChainNetwork.dev // User your desired network + mut deployer := tfgrid.new_deployer(mnemonics, chain_network, mut logger)! + node_twin_id := deployer.client.get_node_twin(node_id)! + zos_version := deployer.client.get_zos_version(node_twin_id)! + deployer.logger.info('Zos version is: ${zos_version}') +} + +fn main() { + test_get_zos_version(u32(14)) or { println('error happened: ${err}') } +} diff --git a/examples/threefold/grid/vm_example.vsh b/examples/threefold/grid/vm_example.vsh new file mode 100644 index 00000000..e91a2bf7 --- /dev/null +++ b/examples/threefold/grid/vm_example.vsh @@ -0,0 +1,40 @@ + + +struct VMSpecs{ + deployment_name string + name string + nodeid string + pub_sshkeys []string + flist string //if any, if used then ostype not used + ostype OSType +} + +enum OSType{ + ubuntu_22_04 + ubuntu_24_04 + arch + alpine +} + +struct VMDeployed{ + name string + nodeid string + //size .. + guid string + yggdrasil_ip string + mycelium_ip string + +} + + +pub fn (vm VMDeployed) builder_node() builder.Node { + +} + +//only connect to yggdrasil and mycelium +// +fn vm_deploy(args_ VMSpecs) VMDeployed{ + + deploymentstate_db.set(args.deployment_name,"vm_${args.name}",VMDeployed.json) + +} \ No newline at end of file diff --git a/examples/threefold/grid/vm_query_example.vsh b/examples/threefold/grid/vm_query_example.vsh new file mode 100644 index 00000000..8145b790 --- /dev/null +++ b/examples/threefold/grid/vm_query_example.vsh @@ -0,0 +1,38 @@ + + +struct NodeQuery{ + location string //how to define location + capacity_available_hdd_gb int + capacity_available_ssd_gb int + capacity_available_mem_gb int + capacity_available_vcpu int //vcpu core's + capacity_free_hdd_gb int + capacity_free_ssd_gb int + capacity_free_mem_gb int + capacity_free_vcpu int //vcpu core's + uptime_min int = 70 //0..99 + bw_min_mb_sec int = 0 //bandwith in mbit per second, min + +} + + +struct NodeInfo{ + location string //how to define location + capacity_available_hdd_gb int + capacity_available_ssd_gb int + capacity_available_mem_gb int + capacity_available_vcpu int //vcpu core's + capacity_free_hdd_gb int + capacity_free_ssd_gb int + capacity_free_mem_gb int + capacity_free_vcpu int //vcpu core's + uptime_min int = 70 //0..99 + bw_min_mb_sec int = 0 //bandwith in mbit per second, min + guid str + ... +} + + +fn node_find(args_ NodeQuery) []NodeInfo{ + +} \ No newline at end of file diff --git a/examples/threefold/grid/webgw_example.vsh b/examples/threefold/grid/webgw_example.vsh new file mode 100644 index 00000000..29316058 --- /dev/null +++ b/examples/threefold/grid/webgw_example.vsh @@ -0,0 +1,18 @@ + + + + + +struct WebGWArgs{ + deployment_name string + //... + + +} + + + +//connect domain name, or exising to it +fn webgateway_rule_deploy(args_ WebGWArgs) []VMDeployed{ + +} \ No newline at end of file diff --git a/examples/threefold/grid/zdb_example.vsh b/examples/threefold/grid/zdb_example.vsh new file mode 100644 index 00000000..9486e6af --- /dev/null +++ b/examples/threefold/grid/zdb_example.vsh @@ -0,0 +1,30 @@ + + +struct ZDBSpecs{ + deployment_name string + nodeid string + namespace string + secret string +} + +struct ZDBDeployed{ + nodeid string + namespace string + secret string +} + + +//test zdb is answering +pub fn (vm ZDBDeployed) ping() bool { + +} + +pub fn (vm ZDBDeployed) redisclient() redisclient... { + +} + +//only connect to yggdrasil and mycelium +// +fn zdb_deploy(args_ ZDBSpecs) ZDBDeployed{ + +} \ No newline at end of file diff --git a/examples/threefold/gridproxy/bill.vsh b/examples/threefold/gridproxy/bill.vsh new file mode 100755 index 00000000..dffffda0 --- /dev/null +++ b/examples/threefold/gridproxy/bill.vsh @@ -0,0 +1,10 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.ui.console + +contract_id := u64(119450) +mut gp_client := gridproxy.new(net:.dev, cache:false)! +bills := gp_client.get_contract_hourly_bill(contract_id)! + +console.print_debug("${bills}") diff --git a/examples/threefold/gridproxy/contract.vsh b/examples/threefold/gridproxy/contract.vsh new file mode 100755 index 00000000..2c5e85cd --- /dev/null +++ b/examples/threefold/gridproxy/contract.vsh @@ -0,0 +1,49 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid as tfgrid +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.ui.console + +fn get_contracts_example() ! { + mut myfilter := gridproxy.contractfilter()! + + myfilter.state = 'Created' + myfilter.contract_type = 'node' + myfilter.twin_id = u64(5191) + + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mycontracts := gp_client.get_contracts(myfilter)! + + console.print_debug("${mycontracts}") +} + +fn get_contract_by_id_example(contract_id u64) ! { + mut myfilter := gridproxy.contractfilter()! + + myfilter.contract_id = contract_id + + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mycontracts := gp_client.get_contracts(myfilter)! + + console.print_debug("${mycontracts}") +} + +fn get_my_contracts_example() ! { + mnemonics := tfgrid.get_mnemonics()! + mut deployer := tfgrid.new_deployer(mnemonics, .dev)! + + mut myfilter := gridproxy.contractfilter()! + + myfilter.twin_id = u64(deployer.twin_id) + myfilter.state = 'created' + + mut gp_client := gridproxy.new(net:.dev, cache:false)! + mycontracts := gp_client.get_contracts(myfilter)! + + console.print_debug('${mycontracts}') +} + +get_contracts_example()! +get_contract_by_id_example(u64(49268))! +get_my_contracts_example()! + diff --git a/examples/threefold/gridproxy/farm.vsh b/examples/threefold/gridproxy/farm.vsh new file mode 100755 index 00000000..66e335b3 --- /dev/null +++ b/examples/threefold/gridproxy/farm.vsh @@ -0,0 +1,30 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.ui.console + +fn get_farms_example() ! { + mut myfilter := gridproxy.farmfilter()! + + myfilter.country = 'Egypt' + myfilter.total_ips = u64(10) + + mut gp_client := gridproxy.new(net:.dev, cache:true)! + myfarms := gp_client.get_farms(myfilter)! + + console.print_debug("${myfarms}") +} + +fn get_farm_by_name_example(farm_name string) ! { + mut myfilter := gridproxy.farmfilter()! + + myfilter.name = farm_name + + mut gp_client := gridproxy.new(net:.main, cache:true)! + myfarms := gp_client.get_farms(myfilter)! + + console.print_debug("${myfarms}") +} + +get_farms_example()! +get_farm_by_name_example("freefarm")! \ No newline at end of file diff --git a/examples/threefold/gridproxy/gateway.vsh b/examples/threefold/gridproxy/gateway.vsh new file mode 100755 index 00000000..55727dfa --- /dev/null +++ b/examples/threefold/gridproxy/gateway.vsh @@ -0,0 +1,30 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.ui.console + +fn get_gateway_nodes_example() ! { + mut myfilter := gridproxy.nodefilter()! + + myfilter.status = 'up' + + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mygateways := gp_client.get_gateways(myfilter)! + + console.print_debug("${mygateways}") + console.print_debug("${mygateways.len}") +} + +fn get_gateway_by_id_example(node_id u64) ! { + mut myfilter := gridproxy.nodefilter()! + + myfilter.node_id = node_id + + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mygateways := gp_client.get_gateways(myfilter)! + + console.print_debug("${mygateways}") +} + +get_gateway_nodes_example()! +get_gateway_by_id_example(u64(11))! \ No newline at end of file diff --git a/examples/threefold/gridproxy/grid.vsh b/examples/threefold/gridproxy/grid.vsh new file mode 100755 index 00000000..2c4da30b --- /dev/null +++ b/examples/threefold/gridproxy/grid.vsh @@ -0,0 +1,31 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.ui.console + +mut gp_client := gridproxy.new(net:.test, cache:true)! + +// get twin list +twins := gp_client.get_twins()! +console.print_debug('${twins}') + +// get farm list +farms := gp_client.get_farms()! +console.print_debug('${farms}') + +// get node list +nodes := gp_client.get_nodes()! +console.print_debug('${nodes}') + +// get gateway list +gateways := gp_client.get_gateways()! +console.print_debug('${gateways}') + +// get contract list +contracts := gp_client.get_contracts()! +console.print_debug('${contracts}') + +// get grid stats +stats := gp_client.get_stats()! +console.print_debug('${stats}') + diff --git a/examples/threefold/gridproxy/node.vsh b/examples/threefold/gridproxy/node.vsh new file mode 100755 index 00000000..03e5b74b --- /dev/null +++ b/examples/threefold/gridproxy/node.vsh @@ -0,0 +1,95 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.ui.console + +fn get_nodes_example() ! { + mut myfilter := gridproxy.nodefilter()! + + myfilter.status = 'up' + myfilter.country = 'belgium' + + mut gp_client := gridproxy.new(net:.main, cache:true)! + mynodes := gp_client.get_nodes(myfilter)! + + console.print_debug("${mynodes}") +} + +fn get_node_by_id_example(node_id u64) ! { + mut myfilter := gridproxy.nodefilter()! + + myfilter.node_id = node_id + + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mynodes := gp_client.get_nodes(myfilter)! + + console.print_debug("${mynodes}") + + // get node available resources + node_available_resources := mynodes[0].calc_available_resources() + console.print_debug("${node_available_resources}") +} + +fn get_node_stats_by_id_example(node_id u64) ! { + mut gp_client := gridproxy.new(net:.dev, cache:true)! + + node_stats := gp_client.get_node_stats_by_id(node_id)! + + console.print_debug("${node_stats}") +} + +fn get_node_by_available_capacity_example() ! { + mut myfilter := gridproxy.nodefilter()! + + // minimum free capacity + myfilter.free_mru = u64(0) + myfilter.free_sru = u64(1024) // 1 tb + myfilter.free_hru = u64(0) + myfilter.free_ips = u64(1) + + // init gridproxy client on devnet with redis cash + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mynodes := gp_client.get_nodes(myfilter)! + + console.print_debug("${mynodes}") +} + +fn get_node_by_city_country_example() ! { + mut myfilter := gridproxy.nodefilter()! + + myfilter.city = 'Rio de Janeiro' + myfilter.country = 'Brazil' + + mut gp_client := gridproxy.new(net:.main, cache:false)! + mynodes := gp_client.get_nodes(myfilter)! + + console.print_debug("${mynodes}") +} + +fn get_node_box_poc_example() ! { + mut myfilter := gridproxy.nodefilter()! + + myfilter.status = 'up' + + mut gp_client := gridproxy.new(net:.main, cache:true)! + mynodes := gp_client.get_nodes(myfilter)! + + for node in mynodes{ + console.print_debug('${node}') + console.print_debug('${node.capacity.total_resources.hru.to_gigabytes()}') + + node_available_resources := node.calc_available_resources() + console.print_debug('${node_available_resources}') + + node_stats := gp_client.get_node_stats_by_id(node.node_id)! + console.print_debug('${node_stats}') + + } +} + +get_nodes_example()! +get_node_by_id_example(u64(11))! +get_node_stats_by_id_example(u64(11))! +get_node_by_available_capacity_example()! +get_node_by_city_country_example()! +get_node_box_poc_example()! \ No newline at end of file diff --git a/examples/threefold/gridproxy/stats.vsh b/examples/threefold/gridproxy/stats.vsh new file mode 100755 index 00000000..1c75a3d5 --- /dev/null +++ b/examples/threefold/gridproxy/stats.vsh @@ -0,0 +1,30 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.threefold.gridproxy.model { NodeStatus } +import freeflowuniverse.herolib.ui.console + +fn get_online_grid_stats_example() ! { + mut myfilter := gridproxy.statfilter()! + + myfilter.status = NodeStatus.online + + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mystats := gp_client.get_stats(myfilter)! + + console.print_debug("${mystats}") +} + +fn get_all_grid_stats_example() ! { + mut myfilter := gridproxy.statfilter()! + + myfilter.status = NodeStatus.all + + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mystats := gp_client.get_stats(myfilter)! + + console.print_debug("${mystats}") +} + +get_online_grid_stats_example()! +get_all_grid_stats_example()! diff --git a/examples/threefold/gridproxy/twin.vsh b/examples/threefold/gridproxy/twin.vsh new file mode 100755 index 00000000..3319910f --- /dev/null +++ b/examples/threefold/gridproxy/twin.vsh @@ -0,0 +1,25 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.ui.console + +fn get_all_twins_example() ! { + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mytwins := gp_client.get_twins()! + + console.print_debug("${mytwins}") +} + +fn get_twin_by_id_example(twin_id u64) ! { + mut myfilter := gridproxy.twinfilter()! + + myfilter.twin_id = twin_id + + mut gp_client := gridproxy.new(net:.dev, cache:true)! + mytwins := gp_client.get_twins(myfilter)! + + console.print_debug("${mytwins}") +} + +get_all_twins_example()! +get_twin_by_id_example(u64(800))! diff --git a/examples/threefold/holochain/.gitignore b/examples/threefold/holochain/.gitignore new file mode 100644 index 00000000..b8f9672c --- /dev/null +++ b/examples/threefold/holochain/.gitignore @@ -0,0 +1,2 @@ +holochain_deployer +holochain_vms diff --git a/examples/threefold/holochain/holochain_deployer.vsh b/examples/threefold/holochain/holochain_deployer.vsh new file mode 100755 index 00000000..b7c9809c --- /dev/null +++ b/examples/threefold/holochain/holochain_deployer.vsh @@ -0,0 +1,67 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.tfrobot +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.clients.dagu + +console.print_header("Deploy test of vmachines on TFGrid using TFRobot.") + +mut bot := tfrobot.new()! + +mut deploy_config := tfrobot.DeployConfig{ + name: 'holotest2' + network: .main + debug: true + node_groups: [ + tfrobot.NodeGroup{ + name: 'hologroup2' + nodes_count: 20 + free_cpu: 8 + free_mru: 8 + free_ssd: 100 + // region:"europe" + }, + ] + vms: [ + tfrobot.VMConfig{ + name: 'myvm' + vms_count: 5 + cpu: 1 + mem: 1 + entry_point: '/sbin/zinit init' + // flist: 'https://hub.grid.tf/mariobassem1.3bot/threefolddev-holochain-latest.flist' + flist: 'https://hub.grid.tf/ashraf.3bot/threefolddev-holochain-latest.flist' + env_vars: { + 'CODE_SERVER_PASSWORD': 'planetfirst' + 'DAGU_BASICAUTH_USERNAME': 'admin' + 'DAGU_BASICAUTH_PASSWORD': 'planetfirst' + } + public_ip4: false + root_size: 50 + planetary: true + }, + ] +} + +//ssh-key is not specified, first key will be chosen + +//DEAL WITH SSH KEYS +tfrobot.sshagent_keys_add(mut deploy_config)! + +console.print_header("nr of ssh keys in ssh-agent:${deploy_config.ssh_keys.len}") + +res := bot.deploy(deploy_config)! + + +console.print_header("Get VM's and ssh into it.") + +for vm in tfrobot.vms_get('holotest')!{ + console.print_debug(vm.str()) + mut node:=vm.node()! + r:=node.exec(cmd:"ls /")! + println(r) +} + + + + diff --git a/examples/threefold/holochain/holochain_vms.vsh b/examples/threefold/holochain/holochain_vms.vsh new file mode 100755 index 00000000..02357d3c --- /dev/null +++ b/examples/threefold/holochain/holochain_vms.vsh @@ -0,0 +1,13 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.tfrobot +import freeflowuniverse.herolib.ui.console + +console.print_header("Get VM's.") + +for vm in tfrobot.vms_get('holotest2')!{ + console.print_debug(vm.str()) + mut node:=vm.node()! + r:=node.exec(cmd:"ls /")! + println(r) +} diff --git a/examples/threefold/holochain/readme.md b/examples/threefold/holochain/readme.md new file mode 100644 index 00000000..c8d773d6 --- /dev/null +++ b/examples/threefold/holochain/readme.md @@ -0,0 +1,82 @@ +# Holochain Example + +Mass deploys Holochain nodes with: +- codeserver +- dagu & server +- ssh +- mycelium +- yggdrasil + +## How to use + +The tfrobot requires Threefold Grid mneumonics so export them before running the example. +Depending on which network tfrobot is configured to, use corresponding. (default is main) + +```bash +export TFGRID_MNEMONIC="jelly fork ..." +``` + +Run the `holochain_deployer.vsh` script. This will deploy VMs. You can interact with the vms over: +- ssh +- codeserver at port 8080 +- dagu api & ui at port 8081 + +go to ipaddress on port 8080 for seeing the codeserver. +e.g. http://[300:463c:9082:e2d6:413d:46e0:c6b1:3de2]:8080/ + +### Credentials + +The credentials for the codeserver and dagu server can be set using the following environment variables. The default values for each variable is set below. + +```bash +CODE_SERVER_PASSWORD=password # password for code server +DAGU_BASICAUTH_USERNAME=admin # dagu ui and api username +DAGU_BASICAUTH_PASSWORD=password # dagu ui and api password +``` + +These environment variables can be passed to the vms using the following configuration when calling `tfrobot.deploy()` + +```js +tfrobot.DeployConfig{ + vms: [ + tfrobot.VMConfig{ + env_vars: { + 'CODE_SERVER_PASSWORD': 'mypass' + } + } + ] +} +``` + +## DAG Example + +The `dag_example.vsh` script demonstrates how the dagu client can be used to send and run a DAG on the machine to scaffold and serve a holochain web app. + +To run: + +`./dag_example.vsh ` + +This sends and runs the DAG, which can be monitored from `:8081` + +Once the DAG reaches its final step and the web app starts running, the UI can be accessed by forwarding the following ports: +- `8282`: Holochain web app +- `8888`: Conductors send hello interface (optional) +- Two conductor ports which are outputted by the final step of the DAG + +Once these ports are forwarded, the Holochain web app can be accessed at `localhost:8282`, and will display the holo network with two conductors. + +# Phase 2 + +- each vm will start sshserver and a call backhome +- call back home is message over mycelium to the originator + - download/install hero as part of the init or hero in flist + - 'hero callhome $comma_separate_ipv6_list_mycelium' + - the hero call home will keep on trying untill it got confirmation that it was received (for 24 h) + - in other words sends alive message to the TFRobot node who deployed the VM + - TFRobot node runs vscript to do deploy, this vscript in loop keeps on showing progress of deploy as well as polling over mycelium if the alive message arrived +- installed mycelium + + +### ideas for later + +- integrate with https://dagu.readthedocs.io/en/latest/web_interface.html \ No newline at end of file diff --git a/examples/threefold/holochain/tasker_example.vsh b/examples/threefold/holochain/tasker_example.vsh new file mode 100755 index 00000000..c8affa9b --- /dev/null +++ b/examples/threefold/holochain/tasker_example.vsh @@ -0,0 +1,50 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.tfrobot +import freeflowuniverse.herolib.ui.console + +console.print_header("Tasker Example.") + +mut vm := tfrobot.vm_get('holotest','test2')! + +//get a set of tasks as we want to execute on the vm +mut tasks:=vm.tasks_new(name:'holochain_scaffold') + +tasks.step_add( + nr:1 + name: 'Verify installation' + command: 'nix run --refresh -j0 -v github:holochain/holochain#hc-scaffold -- --version' + )! + +tasks.step_add( + nr:2 + name: 'Create working directory' + command: 'mkdir -p /root/Holochain' + depends: "1" + )! + +tasks.step_add( + nr:3 + name: 'Scaffold application' + description: 'Scaffold a simple "Hello, World!" Holochain application' + dir: '/root/Holochain' + script: 'nix run github:holochain/holochain#hc-scaffold -- example hello-world || true' + depends: "2" + continue_on_error: true + )! + +tasks.step_add( + nr:4 + name: 'Run Application' + dir: '/root/Holochain/hello-world' + command: 'nix develop --command bash -c "npm install && npm run start" && exit' + depends: "3" + )! + + +vm.tasks_run(tasks)! +vm.tasks_see(tasks)! + +vm.vscode_holochain()! + +vm.vscode_holochain_proxy()! diff --git a/examples/threefold/holochain/tasker_example2.vsh b/examples/threefold/holochain/tasker_example2.vsh new file mode 100755 index 00000000..4e99a1b9 --- /dev/null +++ b/examples/threefold/holochain/tasker_example2.vsh @@ -0,0 +1,29 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.tfrobot +import freeflowuniverse.herolib.ui.console + +console.print_header("Tasker Example 2.") + +mut vm := tfrobot.vm_get('holotest','test1')! + +//get a set of tasks as we want to execute on the vm +mut tasks:=vm.tasks_new(name:'sysadmin') + +tasks.step_add( + nr:1 + name: 'ls' + command: 'ls /' + )! + +tasks.step_add( + nr:2 + name: 'install something' + command: 'nix-env --install mc' + depends: "1" + )! + + +vm.tasks_run(tasks)! +vm.tasks_see(tasks)! + diff --git a/examples/threefold/solana/seahorse_install.sh b/examples/threefold/solana/seahorse_install.sh new file mode 100755 index 00000000..3c740865 --- /dev/null +++ b/examples/threefold/solana/seahorse_install.sh @@ -0,0 +1,263 @@ +#!/bin/bash + +echo "Starting Seahorse installation..." + +# Update and upgrade the system +apt update +apt upgrade -y +apt install -y curl +apt install -y pkg-config build-essential libudev-dev + +# Function to check if a command exists +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +get_latest_github_release() { + local repo=$1 + local api_url="https://api.github.com/repos/${repo}/releases/latest" + local latest_version + + if command -v curl >/dev/null 2>&1; then + latest_version=$(curl -sSf "$api_url" | grep -oP '"tag_name": "\K(.*)(?=")') + elif command -v wget >/dev/null 2>&1; then + latest_version=$(wget -qO- "$api_url" | grep -oP '"tag_name": "\K(.*)(?=")') + else + echo "Error: Neither curl nor wget is available." >&2 + return 1 + fi + + if [ -z "$latest_version" ]; then + echo "Error: Could not fetch the latest version." >&2 + return 1 + fi + + echo "$latest_version" +} + +# Install/Update Rust +install_or_update_rust() { + local latest_version=$(get_latest_github_release "rust-lang/rust") + echo "Rust latest version is $latest_version" + + if ! command_exists rustc; then + echo "Installing Rust..." + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source $HOME/.cargo/env + else + local current_version=$(rustc --version | awk '{print $2}') + + # Remove the 'rust-' prefix from the version string + latest_version=${latest_version#rust-} + + if [ "$current_version" != "$latest_version" ]; then + echo "Updating Rust from version $current_version to $latest_version..." + rustup update stable + updated_version=$(rustc --version | awk '{print $2}') + echo "Rust updated to version $updated_version" + else + echo "Rust is already at the latest version ($current_version)." + fi + fi +} + +# Install rustfmt +install_rustfmc() { + if ! rustup component list | grep -q "rustfmt"; then + echo "Installing rustfmt..." + rustup component add rustfmt + else + echo "rustfmt is installed. Updating Rust will keep it up to date." + fi +} + +# Install/Update Solana +install_or_update_solana() { + local latest_version=$(get_latest_github_release "solana-labs/solana") + + # Remove the 'v' prefix from the version string if present + latest_version=${latest_version#v} + echo "Solana latest version is $latest_version" + + if ! command_exists solana; then + echo "Solana is not installed" + echo "Installing Solana version $latest_version..." + sh -c "$(curl -sSfL https://release.solana.com/v$latest_version/install)" + export PATH="$PATH:$HOME/.local/share/solana/install/active_release/bin" + echo "Solana $latest_version has been installed." + else + local current_version=$(solana --version | awk '{print $2}') + + if [ "$current_version" != "$latest_version" ]; then + echo "Updating Solana from version $current_version to $latest_version..." + sh -c "$(curl -sSfL https://release.solana.com/v$latest_version/install)" + updated_version=$(solana --version | awk '{print $2}') + echo "Solana updated to version $updated_version" + else + echo "Solana is already at the latest version ($current_version)." + fi + fi +} + +# Function to get the latest Node.js version +get_latest_node_version() { + curl -sL https://nodejs.org/dist/index.json | awk -F'"' '/^{/{print $4}' | sed 's/^v//' | head -n 1 +} + +# Install or update Node.js and npm +install_or_update_node_js_npm() { + local latest_version=$(get_latest_node_version) + echo "Node.js latest version is $latest_version" + + if ! command_exists node; then + echo "Installing Node.js version $latest_version..." + curl -fsSL https://deb.nodesource.com/setup_current.x | bash - + apt-get install -y nodejs + else + local current_version=$(node -v | cut -d 'v' -f 2) + if [ "$current_version" != "$latest_version" ]; then + echo "Updating Node.js from version $current_version to $latest_version..." + curl -fsSL https://deb.nodesource.com/setup_current.x | bash - + apt-get install -y nodejs + else + echo "Node.js is already at the latest version ($current_version)." + fi + fi +} + +# Function to get the latest Yarn version +get_latest_yarn_version() { + curl -sS https://api.github.com/repos/yarnpkg/yarn/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sed 's/^v//' +} + +# Install or update Yarn +install_or_update_yarn() { + local latest_version=$(get_latest_yarn_version) + echo "Yarn latest version is $latest_version" + + if ! command_exists yarn; then + echo "Installing Yarn..." + npm install -g yarn + else + local current_version=$(yarn --version) + + if [ "$current_version" != "$latest_version" ]; then + echo "Updating Yarn from $current_version to $latest_version..." + npm install -g yarn@latest + else + echo "Yarn is already up to date (version $current_version)" + fi + fi +} + +# Function to get the latest Anchor version +get_latest_anchor_version() { + curl -sS https://api.github.com/repos/coral-xyz/anchor/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sed 's/^v//' +} + +# Install or update Anchor +install_or_update_anchor() { + local latest_version=$(get_latest_anchor_version) + echo "Anchor latest version is $latest_version" + + if ! command_exists anchor; then + echo "Installing Anchor... (may take a while)" + cargo install --git https://github.com/coral-xyz/anchor avm --locked --force + avm install latest + avm use latest + # echo 'export PATH="$PATH:$HOME/.avm/bin"' >> ~/.bashrc + else + local current_version=$(anchor --version | awk '{print $2}') + + if [ "$current_version" != "$latest_version" ]; then + echo "Updating Anchor from $current_version to $latest_version... (may take a while)" + avm update + avm install latest + avm use latest + else + echo "Anchor is already up to date (version $current_version)" + fi + fi +} + +# Function to get the latest Seahorse version +get_latest_seahorse_version() { + cargo search seahorse-lang --limit 1 | awk '/^seahorse-lang/ {print $3}' | tr -d '"' +} + +# Function to get the current Seahorse version +get_current_seahorse_version() { + seahorse --version 2>&1 | awk '{print $2}' +} + +# Install or update Seahorse +install_or_update_seahorse() { + local latest_version=$(get_latest_seahorse_version) + echo "Seahorse latest version is $latest_version" + + if ! command_exists seahorse; then + echo "Installing Seahorse... (may take a while)" + cargo install seahorse-lang + else + local current_version=$(get_current_seahorse_version) + + if [ "$current_version" != "$latest_version" ]; then + echo "Updating Seahorse from $current_version to $latest_version... (may take a while)" + cargo install seahorse-lang --force + else + echo "Seahorse is already up to date (version $current_version)" + fi + fi +} + +# Run the installation or update processes +install_or_update_rust +install_rustfmc +install_or_update_solana +install_or_update_node_js_npm +install_or_update_yarn +install_or_update_anchor +install_or_update_seahorse + +check_install() { + local command="$1" + local success_message="$2" + local failure_message="$3" + + # ANSI color codes + local GREEN='\033[0;32m' + local RED='\033[0;31m' + local NC='\033[0m' # No Color + + # Run the command and capture its output + local output + output=$(eval "$command" 2>&1) + local exit_code=$? + + if [ $exit_code -eq 0 ]; then + if [ -n "$output" ]; then + echo -e "${GREEN}${success_message}${NC} ${output}" + else + echo -e "${GREEN}${success_message}${NC}" + fi + else + echo -e "${RED}${failure_message}${NC}" + fi + + return $exit_code +} + +# Verify installations +echo "" +echo "---------- Verify Installations ----------" +check_install "rustc --version" "[+] rustc is installed" "[-] rustc is not installed" +check_install "rustfmt --version" "[+] rustfmt is installed" "[-] rustfmt is not installed" +check_install "solana --version" "[+] solana is installed" "[-] solana is not installed" +check_install "node --version" "[+] node is installed" "[-] node is not installed" +check_install "npm --version" "[+] npm is installed" "[-] npm is not installed" +check_install "yarn --version" "[+] yarn is installed" "[-] yarn is not installed" +check_install "anchor --version" "[+] anchor is installed" "[-] anchor is not installed" +check_install "seahorse -V" "[+] seahorse is installed" "[-] seahorse is not installed" +echo "------------------------------------------" +echo "" +echo "Installation and update complete." diff --git a/examples/threefold/solana/seahorse_vm.vsh b/examples/threefold/solana/seahorse_vm.vsh new file mode 100755 index 00000000..527ea3c4 --- /dev/null +++ b/examples/threefold/solana/seahorse_vm.vsh @@ -0,0 +1,387 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.threefold.grid.models +import freeflowuniverse.herolib.threefold.grid as tfgrid +import freeflowuniverse.herolib.threefold.gridproxy +import time +import flag +import rand +import json +import log +import os +import io + +fn main() { + mut fp := flag.new_flag_parser(os.args) + fp.application('Seahorse dev tool') + fp.version('v0.0.1') + fp.skip_executable() + + mnemonics := fp.string_opt('mnemonic', `m`, 'Your Mnemonic phrase')! + chain_network := fp.string('network', `n`, 'main', 'Your desired chain network (main, test, qa, dev). Defaults to main') + ssh_key := fp.string_opt('ssh_key', `s`, 'Your public ssh key')! + code_server_pass := fp.string('code_server_pass', `p`, 'password', 'Machine code server password. This will be set as a password for the code server on the deployed machine. Defaults to password') + cpu := fp.int('cpu', `c`, 4, 'Machine CPU provisioning. Defaults to 4') + memory := fp.int('ram', `r`, 8, 'Machine memory provisioning in GB. Defaults to 8') + disk := fp.int('disk', `d`, 30, 'Machine Disk space provisioning in GB. Defaults to 30') + public_ip := fp.bool('public_ip', `i`, false, 'True to allow public ip v4') + + mut logger := &log.Log{} + logger.set_level(.debug) + + + // ##### Part 1 ##### + + + nodes=tfgrid.search(...)! //gives error if no nodes + + //default is mycelium + vm:=tfgrid.vm_new(profilename="main",name="myvm",mem_mb=4000,ssd_gb=50,cpu_cores=4,nodeid=nodes[0].id,flist='') + + vm.shell() + vm.ipaddr + + vm.webgw_add(...) + + b:=vm.builder()! + + vm:=tfgrid.vm_get(...) + vm.delete()! + + //gives me a builder (can do ssh on it) + b.exec(..) + + chain_net_enum := get_chain_network(chain_network)! + mut deployer := tfgrid.new_deployer(mnemonics, chain_net_enum, mut logger)! + + mut workloads := []models.Workload{} + // node_id := get_node_id(chain_net_enum, memory, disk, cpu, public_ip)! + node_id := u32(146) + logger.info('deploying on node: ${node_id}') + + + + + network_name := 'net_${rand.string(5).to_lower()}' // autocreate a network + wg_port := deployer.assign_wg_port(node_id)! + mut network := models.Znet{ + ip_range: '10.1.0.0/16' // auto-assign + subnet: '10.1.1.0/24' // auto-assign + wireguard_private_key: 'GDU+cjKrHNJS9fodzjFDzNFl5su3kJXTZ3ipPgUjOUE=' // autocreate + wireguard_listen_port: wg_port + mycelium: models.Mycelium{ + hex_key: rand.string(32).bytes().hex() + } + + } + + workloads << network.to_workload(name: network_name, description: 'test_network1') + + mut public_ip_name := '' + if public_ip{ + public_ip_name = rand.string(5).to_lower() + workloads << models.PublicIP{ + v4: true + }.to_workload(name: public_ip_name) + } + + zmachine := models.Zmachine{ + flist: 'https://hub.grid.tf/petep.3bot/threefolddev-ubuntu24.04-latest.flist' + network: models.ZmachineNetwork{ + interfaces: [ + models.ZNetworkInterface{ + network: network_name + ip: '10.1.1.3' + }, + ] + public_ip: public_ip_name + planetary: true + mycelium: models.MyceliumIP{ + network: network_name + hex_seed: rand.string(6).bytes().hex() + } + } + entrypoint: '/sbin/zinit init' // from user or default + compute_capacity: models.ComputeCapacity{ + cpu: u8(cpu) + memory: i64(memory) * 1024 * 1024 * 1024 + } + size: u64(disk) * 1024 * 1024 * 1024 + env: { + 'SSH_KEY': ssh_key + 'CODE_SERVER_PASSWORD': code_server_pass + } + } + + workloads << zmachine.to_workload( + name: 'vm_${rand.string(5).to_lower()}' + description: 'zmachine_test' + ) + + signature_requirement := models.SignatureRequirement{ + weight_required: 1 + requests: [ + models.SignatureRequest{ + twin_id: deployer.twin_id + weight: 1 + }, + ] + } + + mut deployment := models.new_deployment( + twin_id: deployer.twin_id + description: 'seahorse deployment' + workloads: workloads + signature_requirement: signature_requirement + ) + deployment.add_metadata('vm', 'SeahorseVM') + + contract_id := deployer.deploy(node_id, mut deployment, deployment.metadata, 0) or { + logger.error('failed to deploy deployment: ${err}') + exit(1) + } + logger.info('deployment contract id: ${contract_id}') + dl := deployer.get_deployment(contract_id, node_id) or { + logger.error('failed to get deployment data: ${err}') + exit(1) + } + + // logger.info('deployment:\n${dl}') + machine_res := get_machine_result(dl)! + logger.info('zmachine result: ${machine_res}') + + // Add a small delay after deployment to ensure the VM is fully up and running before attempting to connect + logger.info('Wait for 30 seconds to ensure the VM is fully up...') + time.sleep(30 * time.second) // Wait for 30 seconds + + + // ##### Part 2 ##### + + // Check if mycelium is installed on my local machine + if !is_mycelium_installed() { + logger.error('Mycelium is not installed. Please install Mycelium before proceeding.') + return + } else { + logger.info('Mycelium is installed.') + } + + // Check if mycelium is running on my local machine + if !is_mycelium_running() { + // logger.info('Warning: Mycelium is not running.') + // logger.info('Attempting to start Mycelium...') + // os.execute('sudo mycelium --peers tcp://188.40.132.242:9651 tcp://[2a01:4f8:212:fa6::2]:9651 quic://185.69.166.7:9651 tcp://[2a02:1802:5e:0:8c9e:7dff:fec9:f0d2]:9651 tcp://65.21.231.58:9651 quic://[2a01:4f9:5a:1042::2]:9651') + // // Wait a bit and check again + // time.sleep(5 * time.second) + // if !is_mycelium_running() { + // logger.error('Failed to start Mycelium. Please start it manually before proceeding.') + // return + // } + + logger.error('Mycelium is not running on local machine.') + return + } else { + logger.info('Mycelium is running on local machine.') + } + + remote_mycelium_ip := machine_res.mycelium_ip + logger.info('Mycelium IP: ${remote_mycelium_ip}') + + // Ping remote mycelium ip + if !ping_ip(remote_mycelium_ip, 5) { + logger.error('Failed to ping ${remote_mycelium_ip} after 5 attempts') + return + } else { + logger.info('Successed to ping ${remote_mycelium_ip}') + } + + // Attempt the SSH connection + if !try_ssh_connection(remote_mycelium_ip) { + logger.error('Unable to establish SSH connection. Please check your network and VM status.') + return + } else { + logger.info('Ready to proceed with further operations') + } + + // Run installation script on remote VM + seahorse_install_script := 'seahorse_install.sh' + if !execute_remote_script(remote_mycelium_ip, seahorse_install_script) { + logger.error('Seahorse remote installation failed') + return + } else { + logger.info('Seahorse remote installation completed successfully') + } +} + +fn get_machine_result(dl models.Deployment) !models.ZmachineResult { + for _, w in dl.workloads { + if w.type_ == models.workload_types.zmachine { + res := json.decode(models.ZmachineResult, w.result.data)! + return res + } + } + + return error('failed to get zmachine workload') +} + +fn get_chain_network(network string) !tfgrid.ChainNetwork { + chain_net_enum := match network { + 'dev' { tfgrid.ChainNetwork.dev } + 'qa' { tfgrid.ChainNetwork.qa } + 'test' { tfgrid.ChainNetwork.test } + 'main' { tfgrid.ChainNetwork.main } + else { return error('invalid chain newtork ${network}. must be one of (dev, qa, test, main)') } + } + + return chain_net_enum +} + +fn get_node_id(network tfgrid.ChainNetwork, memory int, disk int, cpu int, public_ip bool) !u32{ + gp_net := match network { + .dev { gridproxy.TFGridNet.dev } + .qa { gridproxy.TFGridNet.qa } + .test { gridproxy.TFGridNet.test } + .main { gridproxy.TFGridNet.main } + } + + mut gridproxy_client := gridproxy.get(gp_net, false)! + mut free_ips := u64(0) + if public_ip{ + free_ips = 1 + } + + mut node_it := gridproxy_client.get_nodes_has_resources( + free_mru_gb: u64(memory) + free_sru_gb: u64(disk) + free_cpu: u64(cpu) + free_ips: free_ips + ) + nodes := node_it.next() + mut node_id := u32(0) // get from user or use gridproxy to get nodeid + if nodes_list := nodes { + node_id = u32(nodes_list[0].node_id) + } else { + return error('cannot find a suitable node matching your specs') + } + + return node_id +} + + +// Function to check if Mycelium is installed +fn is_mycelium_installed() bool { + result := os.execute('mycelium --version') + return result.exit_code == 0 +} + +// Function to check if Mycelium is running locally +fn is_mycelium_running() bool { + mut logger := &log.Log{} + logger.set_level(.debug) + + // Use pgrep to find Mycelium processes + result := os.execute('pgrep -f "^mycelium\\s"') + + if result.exit_code != 0 { + logger.debug('No Mycelium process found') + return false + } + + pids := result.output.trim_space().split('\n') + logger.info('Mycelium process IDs: ${pids}') + + return pids.len > 0 +} + +fn ping_ip(ip string, attempts int) bool { + for i := 0; i < attempts; i++ { + result := os.execute('ping6 -c 1 -W 2 $ip') + if result.exit_code == 0 { + return true + } + time.sleep(1 * time.second) + } + return false +} + +fn try_ssh_connection(mycelium_ip string) bool { + mut logger := &log.Log{} + logger.set_level(.debug) + + logger.info('Attempting SSH connection...') + + // Use -6 flag to force IPv6 + command := 'ssh -6 -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${mycelium_ip} true' + + logger.info('Executing SSH command: ${command}') + result := os.execute(command) + + if result.exit_code == 0 { + logger.info('SSH connection successful') + return true + } else { + logger.info('SSH connection failed: ${result.output}') + return false + } +} + +fn execute_remote_script(mycelium_ip string, script_name string) bool { + mut logger := &log.Log{} + logger.set_level(.info) + + // Get the directory of the V script + v_script_dir := os.dir(os.executable()) + logger.info('V script directory: ${v_script_dir}') + + // Construct the full path to the install script + script_path := os.join_path(v_script_dir, script_name) + logger.info('Full script path: ${script_path}') + + // Ensure the script exists + if !os.exists(script_path) { + logger.error('Script ${script_path} not found') + return false + } + + // Format the IPv6 address correctly for SSH and SCP commands + ssh_ip := mycelium_ip + scp_ip := if mycelium_ip.contains(':') { '[${mycelium_ip}]' } else { mycelium_ip } + + remote_script_path := '/tmp/${script_name}' + + // Construct the SSH and SCP commands + scp_command := 'scp -6 -o StrictHostKeyChecking=no ${script_path} root@${scp_ip}:${remote_script_path}' + ssh_command := 'ssh -6 -o ConnectTimeout=10 -o StrictHostKeyChecking=no -tt root@${ssh_ip}' + + // Copy the script to the remote machine + logger.info('Copying script to remote machine: ${scp_command}') + scp_result := os.execute(scp_command) + if scp_result.exit_code != 0 { + logger.error('Failed to copy script. Exit code: ${scp_result.exit_code}') + logger.error('SCP output: ${scp_result.output}') + return false + } + + // Verify if the script was copied successfully + check_file_command := '${ssh_command} "ls -l ${remote_script_path}"' + check_result := os.execute(check_file_command) + if check_result.exit_code != 0 { + logger.error('Failed to verify script on remote machine. Exit code: ${check_result.exit_code}') + return false + } + logger.info('Script found on remote machine: ${remote_script_path}') + + // Now execute the script on the remote machine and stream the output + log_file := '/tmp/output.log' + run_script_command := '${ssh_command} "bash -l ${remote_script_path} | tee ${log_file}"' + logger.info('Executing remote script: ${run_script_command}') + logger.info('Follow remote script execution: ${ssh_command} "tail -f ${log_file}"') + run_result := os.execute(run_script_command) + logger.info('See full output log file: ${ssh_command} "cat ${log_file}"') + if run_result.exit_code == 0 { + logger.info('Remote script execution completed successfully') + return true + } else { + logger.error('Remote script execution failed with exit code: ${run_result.exit_code}') + return false + } +} \ No newline at end of file diff --git a/examples/threefold/tfgrid3deployer/heroscript/vms.hero b/examples/threefold/tfgrid3deployer/heroscript/vms.hero new file mode 100644 index 00000000..2a2154cf --- /dev/null +++ b/examples/threefold/tfgrid3deployer/heroscript/vms.hero @@ -0,0 +1,19 @@ + + +!!tfgriddeploy.define_vm_requirement + name: 'vm_req_1' + description: 'Requirements for a test VM on the TF Grid' + cpu: 4 //cores + memory: 8 //gbyte of node + public_ip4: true + public_ip6: false + planetary: true + mycelium: false + nodes: '1,2,3' + network_requirement:'net_req_1' //can be empty + +!!tfgriddeploy.define_network_requirement + name: 'net_req_1' + ip_range: '192.168.0.0/24' + subnet: '255.255.255.0' + diff --git a/examples/threefold/tfgrid3deployer/tfgrid3deployer_example.vsh b/examples/threefold/tfgrid3deployer/tfgrid3deployer_example.vsh new file mode 100755 index 00000000..8f855922 --- /dev/null +++ b/examples/threefold/tfgrid3deployer/tfgrid3deployer_example.vsh @@ -0,0 +1,97 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals -cg run + +import freeflowuniverse.herolib.threefold.gridproxy +import freeflowuniverse.herolib.threefold.tfgrid3deployer +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.installers.threefold.griddriver + + +fn main(){ + mut installer:= griddriver.get()! + installer.install()! + + // v := tfgrid3deployer.get()! + // println('cred: ${v}') + // deployment_name := "my_deployment27" + + // // mut deployment := tfgrid3deployer.new_deployment(deployment_name)! + // mut deployment := tfgrid3deployer.get_deployment(deployment_name)! + // // deployment.add_machine(name: "my_vm1" cpu: 1 memory: 2 planetary: true mycelium: tfgrid3deployer.Mycelium{} nodes: [u32(11)]) + // // deployment.add_machine(name: "my_vm3" cpu: 1 memory: 2 planetary: true mycelium: tfgrid3deployer.Mycelium{} nodes: [u32(11)]) + // // deployment.add_machine(name: "my_vm3" cpu: 1 memory: 2 planetary: true mycelium: tfgrid3deployer.Mycelium{} nodes: [u32(28)]) + // // deployment.add_zdb(name: "my_zdb", password: "my_passw&rd", size: 2) + // // deployment.add_webname(name: 'mywebname2', backend: 'http://37.27.132.47:8000') + // // deployment.deploy()! + + // // deployment.add_machine(name: "my_vm2" cpu: 2 memory: 3 planetary: true mycelium: true nodes: [u32(28)]) + // // deployment.deploy()! + + // deployment.remove_machine("my_vm1")! + // // deployment.remove_webname("mywebname2")! + // // deployment.remove_zdb("my_zdb")! + // // deployment.deploy()! + + // // deployment.vm_get("my_vm1")! + + // // // deployment.remove_machine(name: "my_vm121") + // // // deployment.update_machine(name: "my_vm121") + // println("deployment: ${deployment}") + + // If not sent: The client will create a network for the deployment. + // deployment.network = NetworkSpecs{ + // name: 'hamadanetcafe' + // ip_range: '10.10.0.0/16' + // } + + // deployment.add_machine(name: "my_vm121" cpu: 1 memory: 2 planetary: true mycelium: true nodes: [u32(11)]) + // deployment.add_zdb(name: "my_zdb", password: "my_passw&rd", size: 2) + // deployment.add_webname(name: 'mywebname2', backend: 'http://37.27.132.47:8000') + // deployment.add_machine(name: "my_vm1" cpu: 1 memory: 2 planetary: true mycelium: true nodes: [u32(28)]) + // deployment.deploy()! + + // vm1 := deployment.vm_get("my_vm1")! + // reachable := vm1.healthcheck()! + // println("vm reachable: ${reachable}") + + // if !reachable { + // deployment.vm_delete()! + // deployment.vm_deploy()! + // } + + // if !rach { + // vm1.delete()! + // vm1.deploy()! + // } + + + /* + TODO: Agreed on + + # Deploying a new deployemnt + mut deployment := tfgrid3deployer.new_deployment(deployment_name)! + deployment.add_machine(name: "my_vm121" cpu: 1 memory: 2 planetary: true mycelium: true nodes: [u32(11)]) + deployment.add_zdb(name: "my_zdb", password: "my_passw&rd", size: 2) + deployment.deploy()! + + # if the user wants to load the deployment and do some actions on it: + mut deployment := tfgrid3deployer.get_deployment(deployment_name)! + deployment.add_webname(name: 'mywebname2', backend: 'http://37.27.132.47:8000') + deployment.add_machine(name: "my_vm1" cpu: 1 memory: 2 planetary: true mycelium: true nodes: [u32(28)]) + deployment.deploy()! + + # The user wants to delete the recently deployed machine + mut deployment := tfgrid3deployer.get_deployment(deployment_name)! + deployment.remove_machine(name: "my_vm1") + deployment.deploy()! + + # The user wants to update the first deployed machine + mut deployment := tfgrid3deployer.get_deployment(deployment_name)! + deployment.remove_machine(name: "my_vm1") + deployment.add_machine(name: "my_vm1" cpu: 1 memory: 2 planetary: true mycelium: true nodes: [u32(28)]) + deployment.deploy()! + ## PS: The same goes with ZDBs and Webnames + + # How deploy works: + 1. Let's assume the user wants to add one more workload + */ +} diff --git a/examples/tools/imagemagick/.backup/large_png.png.0 b/examples/tools/imagemagick/.backup/large_png.png.0 new file mode 100644 index 00000000..ac2380e2 Binary files /dev/null and b/examples/tools/imagemagick/.backup/large_png.png.0 differ diff --git a/examples/tools/imagemagick/example.v b/examples/tools/imagemagick/example.v new file mode 100644 index 00000000..9d780882 --- /dev/null +++ b/examples/tools/imagemagick/example.v @@ -0,0 +1,27 @@ +module main + +import freeflowuniverse.herolib.conversiontools.imagemagick +import gittools + +fn do() ! { + if !imagemagick.installed() { + panic('need imagemagick') + } + + mut gs := gittools.get(root: '/tmp/code')! + + url := 'https://github.com/threefoldfoundation/www_examplesite/tree/development/src' + mut gr := gs.repo_get_from_url(url: url, pull: false, reset: false)! + gr.remove_changes()! + + path := gr.path_content_get() // path of the manual + + imagemagick.scan(path: path, backupdir: '/tmp/backupimages')! + + // remove changes so we can do again + // gr.remove_changes()! +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/tools/imagemagick/large_png.png b/examples/tools/imagemagick/large_png.png new file mode 100644 index 00000000..ac2380e2 Binary files /dev/null and b/examples/tools/imagemagick/large_png.png differ diff --git a/examples/tools/imagemagick/not_an_img.md b/examples/tools/imagemagick/not_an_img.md new file mode 100644 index 00000000..2c8841e9 --- /dev/null +++ b/examples/tools/imagemagick/not_an_img.md @@ -0,0 +1 @@ +This is not an image diff --git a/examples/tools/imagemagick/small_png.png b/examples/tools/imagemagick/small_png.png new file mode 100644 index 00000000..591bff2c Binary files /dev/null and b/examples/tools/imagemagick/small_png.png differ diff --git a/examples/tools/tmux/examples/.gitignore b/examples/tools/tmux/examples/.gitignore new file mode 100644 index 00000000..b80ce066 --- /dev/null +++ b/examples/tools/tmux/examples/.gitignore @@ -0,0 +1 @@ +tmux diff --git a/examples/tools/tmux/examples/tmux.v b/examples/tools/tmux/examples/tmux.v new file mode 100644 index 00000000..bac1ff97 --- /dev/null +++ b/examples/tools/tmux/examples/tmux.v @@ -0,0 +1,15 @@ +module main + +import freeflowuniverse.herolib.osal.tmux + +fn do() ! { + mut t := tmux.new()! + t.session_delete('main')! + println(t) + t.window_new(name: 'test', cmd: 'mc', reset: true)! + println(t) +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/tools/vault/examples/easy/.vault/meta b/examples/tools/vault/examples/easy/.vault/meta new file mode 100644 index 00000000..e69de29b diff --git a/examples/tools/vault/examples/easy/subdir/.vault/meta b/examples/tools/vault/examples/easy/subdir/.vault/meta new file mode 100644 index 00000000..3d69a3c7 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/.vault/meta @@ -0,0 +1,2 @@ +0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f|1662456738|1|test2.md +87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7|1662456738|1|test.md \ No newline at end of file diff --git a/examples/tools/vault/examples/easy/subdir/.vault/test.1.md b/examples/tools/vault/examples/easy/subdir/.vault/test.1.md new file mode 100644 index 00000000..78981922 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/.vault/test.1.md @@ -0,0 +1 @@ +a diff --git a/examples/tools/vault/examples/easy/subdir/.vault/test2.1.md b/examples/tools/vault/examples/easy/subdir/.vault/test2.1.md new file mode 100644 index 00000000..61780798 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/.vault/test2.1.md @@ -0,0 +1 @@ +b diff --git a/examples/tools/vault/examples/easy/subdir/subsudir/.vault/meta b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/meta new file mode 100644 index 00000000..d4260fd6 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/meta @@ -0,0 +1,5 @@ +a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478|1662456738|1|test3.md +ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39d|1662456824|2|test3.md +e7f6c011776e8db7cd330b54174fd76f7d0216b612387a5ffcfb81e6f0919683|1662456824|3|test3.md +ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39d|1662457271|4|test3.md +ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb|1662457271|5|test3.md \ No newline at end of file diff --git a/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.1.md b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.1.md new file mode 100644 index 00000000..f2ad6c76 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.1.md @@ -0,0 +1 @@ +c diff --git a/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.2.md b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.2.md new file mode 100644 index 00000000..7813681f --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.2.md @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.3.md b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.3.md new file mode 100644 index 00000000..62f94575 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.3.md @@ -0,0 +1 @@ +6 \ No newline at end of file diff --git a/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.4.md b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.4.md new file mode 100644 index 00000000..7813681f --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.4.md @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.5.md b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.5.md new file mode 100644 index 00000000..2e65efe2 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/subsudir/.vault/test3.5.md @@ -0,0 +1 @@ +a \ No newline at end of file diff --git a/examples/tools/vault/examples/easy/subdir/subsudir/test3.md b/examples/tools/vault/examples/easy/subdir/subsudir/test3.md new file mode 100644 index 00000000..2e65efe2 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/subsudir/test3.md @@ -0,0 +1 @@ +a \ No newline at end of file diff --git a/examples/tools/vault/examples/easy/subdir/test.md b/examples/tools/vault/examples/easy/subdir/test.md new file mode 100644 index 00000000..78981922 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/test.md @@ -0,0 +1 @@ +a diff --git a/examples/tools/vault/examples/easy/subdir/test2.md b/examples/tools/vault/examples/easy/subdir/test2.md new file mode 100644 index 00000000..61780798 --- /dev/null +++ b/examples/tools/vault/examples/easy/subdir/test2.md @@ -0,0 +1 @@ +b diff --git a/examples/tools/vault/examples/simple.v b/examples/tools/vault/examples/simple.v new file mode 100644 index 00000000..b0a1fb31 --- /dev/null +++ b/examples/tools/vault/examples/simple.v @@ -0,0 +1,33 @@ +module main + +import freeflowuniverse.herolib.vault +import freeflowuniverse.herolib.core.pathlib +import os + +const testdir2 = os.dir(@FILE) + '/easy' + +fn do() ? { + mut v := vault.do(testdir2)? + + remember := v.hash() + + mut p := pathlib.get('${testdir2}/subdir/subsudir/test3.md') + p.write('5')? + mut v2 := vault.do(testdir2)? // will remember the change + p.write('a')? + mut v3 := vault.do(testdir2)? // will remember the change + + println(v3.superlist()) + println(v3.hash()) + + // restore to the original scan + mut v4 := vault.restore(0)? + remember3 := v.hash() + assert remember == remember3 + + v3.delete()? +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/tools/vault/examples/vault_example.v b/examples/tools/vault/examples/vault_example.v new file mode 100644 index 00000000..bc01e4a5 --- /dev/null +++ b/examples/tools/vault/examples/vault_example.v @@ -0,0 +1,26 @@ +module main + +import freeflowuniverse.herolib.core.pathlib +import freeflowuniverse.herolib.vault +import os + +const testdir = os.dir(@FILE) + '/../../pathlib/examples/test_path' + +fn do() ? { + // just to check it exists + mut p := pathlib.get_dir(testdir, false)? + p.absolute() + println(p) + // will load the vault, doesn't process files yet + // mut vault1 := vault.scan('myvault', mut p)? + // println(vault) + // vault1.delete()? + mut vault2 := vault.scan('myvault', mut p)? + vault2.shelve()? + // println(vault2) + vault2.delete()? +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/ui/console/console2/console2.v b/examples/ui/console/console2/console2.v new file mode 100644 index 00000000..4e78bdb0 --- /dev/null +++ b/examples/ui/console/console2/console2.v @@ -0,0 +1,25 @@ +module main + +import freeflowuniverse.herolib.console + + +fn do() ! { + mut c:=console.new() + r:=c.ask_question(question:"my question")! + println(r) + + r2:=c.ask_dropdown_multiple(question:"my dropdown",items:["a","b","c"])! + println(r2) + + r3:=c.ask_dropdown_multiple(question:"my dropdown",items:["a","b","c"],default:["a","b"],clear:true)! + println(r3) + + r3:=c.ask_dropdown(question:"my dropdown",items:["a","b","c"],default:["c"],clear:true)! + println(r3) + + +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/ui/console/flow1/flow1.v b/examples/ui/console/flow1/flow1.v new file mode 100644 index 00000000..d3f5b59d --- /dev/null +++ b/examples/ui/console/flow1/flow1.v @@ -0,0 +1,54 @@ +module main + +import freeflowuniverse.herolib.ui +import freeflowuniverse.herolib.ui.generic + +struct RoomOrderFlow { +pub mut: + current_product string + ui generic.UserInterface +} + +fn (mut f RoomOrderFlow) room_choice() ! { + i := f.ui.ask_dropdown( + description: 'Which type of room do you want?' + items: ['penthouse', 'normal', 'single', 'appartment_room'] + warning: 'Please select your right type of room' + )! + + println(i) + + smoker := f.ui.ask_yesno(description: 'Are you a smoker?')! + if smoker { + smoke := f.ui.ask_yesno(description: 'Do you want to smoke in your room?')! + if smoke == false { + println('Please realize if we detect you have smoked in your room we will charge 100USD to deep clean the room.') + } + } + if smoker == false { + // TODO check there is a non smoking room. + println("We are very sorry, we didn't find a non smoking room, do you want another room or you are ok.") + } else { + println("We are a non smoking hotel, we're sorry.") + } +} + +fn do() ! { + // all bool // means user can choose all of them + // description string + // items []string + // warning string + // reset bool = true + + gui := ui.new(channel: .console)! + mut f := RoomOrderFlow{ + ui: gui + } + f.room_choice()! + + // println(i) +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/ui/flow1.v b/examples/ui/flow1.v new file mode 100644 index 00000000..b2db9377 --- /dev/null +++ b/examples/ui/flow1.v @@ -0,0 +1,45 @@ +module main + +import freeflowuniverse.herolib.ui + +struct RoomOrderFlow { + current_product string + ui ui.UserInterface +} + +fn (mut f RoomOrderFlow) room_select() ! { + i := f.ui.ask_dropdown_int( + description: 'Which type of room do you want?' + items: ['penthouse', 'normal', 'single', 'appartment_room'] + warning: 'Please select your right type of room' + reset: true + ) + // match + smoker := f.ui.ask_yesno(description: 'Are you a smoker?') + if smoker { + smoke := f.ui.ask_yesno(description: 'Do you want to smoke in your room?') + if smoke == false { + println('Please realize if we detect you have smoked in your room we will charge 100USD to deep clean the room.') + } + } + if smoker == false { + // TODO check there is a non smoking room. + if false { + println("We are very sorry, we didn't find a non smoking room, do you want another room or you are ok.") + } + } +} + +fn do() ! { + // open your flow and attach the required channel to it + mut f := RoomOrderFlow{ + ui: ui.new(.console) + } + f.room_select()! + + // println(i) +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/ui/silence.vsh b/examples/ui/silence.vsh new file mode 100755 index 00000000..4d9ff2ea --- /dev/null +++ b/examples/ui/silence.vsh @@ -0,0 +1,14 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.ui.console + +console.silent_set() +mut job2 := osal.exec(cmd: 'ls /',debug:true)! +println("I got nothing above") + +console.silent_unset() +println("now I will get output") + +osal.exec(cmd: 'ls /',debug:true)! + diff --git a/examples/ui/telegram/flow1.v b/examples/ui/telegram/flow1.v new file mode 100644 index 00000000..6b0773fe --- /dev/null +++ b/examples/ui/telegram/flow1.v @@ -0,0 +1,49 @@ +module main + +import freeflowuniverse.herolib.ui.console + +struct RoomOrderFlow { + current_product string + ui UserInterface +} + +fn (mut f RoomOrderFlow) room_choice() ! { + i := ui.ask_dropdown( + description: 'Which type of room do you want?' + items: ['penthouse', 'normal', 'single', 'appartment_room'] + warning: 'Please select your right type of room' + reset: true + ) + + // match + smoker := console.ask_yesno(description: 'Are you a smoker?') + if smoker { + smoke := console.ask_yesno(description: 'Do you want to smoke in your room?') + if smoke == false { + println('Please realize if we detect you have smoked in your room we will charge 100USD to deep clean the room.') + } + } + if smoker == false { + // TODO check there is a non smoking room. + if false { + println("We are very sorry, we didn't find a non smoking room, do you want another room or you are ok.") + } + } +} + +fn do() ! { + // all bool // means user can choose all of them + // description string + // items []string + // warning string + // reset bool = true + + mut f := RoomOrderFlow{} + f.room_choice()! + + // println(i) +} + +fn main() { + do() or { panic(err) } +} diff --git a/examples/virt/docker/.gitignore b/examples/virt/docker/.gitignore new file mode 100644 index 00000000..5ae2bf86 --- /dev/null +++ b/examples/virt/docker/.gitignore @@ -0,0 +1,7 @@ +docker_registry_example +docker_e1 +docker_dev_tools +docker_init +docker_registry +presearch_docker +tf_dashboard diff --git a/examples/virt/docker/docker_dev_tools.vsh b/examples/virt/docker/docker_dev_tools.vsh new file mode 100644 index 00000000..a82c9af1 --- /dev/null +++ b/examples/virt/docker/docker_dev_tools.vsh @@ -0,0 +1,17 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.virt.docker + +mut engine := docker.new(prefix: '', localonly: true)! + +mut r := engine.recipe_new(name: 'dev_tools', platform: .alpine) + +r.add_from(image: 'alpine', tag: 'latest')! + +r.add_package(name: 'git,vim')! + +r.add_zinit()! + +r.add_sshserver()! + +r.build(true)! diff --git a/examples/virt/docker/docker_init.vsh b/examples/virt/docker/docker_init.vsh new file mode 100644 index 00000000..6a594567 --- /dev/null +++ b/examples/virt/docker/docker_init.vsh @@ -0,0 +1,10 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.virt.docker + +mut engine := docker.new()! + +engine.reset_all()! + +println(engine) + diff --git a/examples/virt/docker/docker_registry.vsh b/examples/virt/docker/docker_registry.vsh new file mode 100644 index 00000000..ccb6eea5 --- /dev/null +++ b/examples/virt/docker/docker_registry.vsh @@ -0,0 +1,13 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.virt.docker + +mut engine := docker.new()! + +// engine.reset_all()! + +dockerhub_datapath := '/Volumes/FAST/DOCKERHUB' + +engine.registry_add(datapath: dockerhub_datapath, ssl: true)! + +println(engine) diff --git a/examples/virt/docker/presearch_docker.vsh b/examples/virt/docker/presearch_docker.vsh new file mode 100644 index 00000000..ade03800 --- /dev/null +++ b/examples/virt/docker/presearch_docker.vsh @@ -0,0 +1,27 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.virt.docker +import os + +registration_code := os.getenv('PRESEARCH_CODE') + +if registration_code == '' { + println("Can't find presearch registration code please run 'export PRESEARCH_CODE=...'") + exit(1) +} + +mut engine := docker.new(prefix: '', localonly: true)! +mut recipe := engine.compose_new(name: 'presearch') +mut presearch_node := recipe.service_new(name: 'presearch_node', image: 'presearch/node')! + +presearch_node.volume_add('/presearch-node-storage', '/app/node')! +presearch_node.env_add('REGISTRATION_CODE', '${registration_code}') + +mut presearch_updater := recipe.service_new( + name: 'presearch_updater' + image: 'presearch/auto-updater' +)! +presearch_updater.volume_add('/var/run/docker.sock', '/var/run/docker.sock')! + +recipe.start()! + diff --git a/examples/virt/docker/tf_dashboard.vsh b/examples/virt/docker/tf_dashboard.vsh new file mode 100644 index 00000000..291b4e89 --- /dev/null +++ b/examples/virt/docker/tf_dashboard.vsh @@ -0,0 +1,41 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run +import freeflowuniverse.herolib.virt.docker + +mut engine := docker.new(prefix: '', localonly: true)! + +mut recipe := engine.recipe_new(name: 'tf_dashboard', platform: .alpine) + +println(' - build dashboard') + +recipe.add_from(image: 'nginx', tag: 'alpine')! + +recipe.add_nodejsbuilder()! + +recipe.add_run(cmd: 'apk add git')! +recipe.add_run(cmd: 'npm i -g yarn')! +recipe.add_run( + cmd: ' + git clone https://github.com/threefoldtech/tfgrid-sdk-ts.git /app + cd /app/packages/dashboard + yarn install + yarn lerna run build --no-private + yarn workspace @threefold/dashboard build +' +)! + +recipe.add_run( + cmd: ' + rm /etc/nginx/conf.d/default.conf + cp /app/packages/dashboard/nginx.conf /etc/nginx/conf.d + apk add --no-cache bash + chmod +x /app/packages/dashboard/scripts/build-env.sh + cp -r /app/packages/dashboard/dist /usr/share/nginx/html +' +)! + +recipe.add_run(cmd: 'echo "daemon off;" >> /etc/nginx/nginx.conf')! +recipe.add_cmd(cmd: '/bin/bash -c /app/packages/dashboard/scripts/build-env.sh')! +recipe.add_entrypoint(cmd: 'nginx')! + +recipe.build(false)! + diff --git a/examples/virt/hetzner/hetzner_example.vsh b/examples/virt/hetzner/hetzner_example.vsh new file mode 100755 index 00000000..569078f5 --- /dev/null +++ b/examples/virt/hetzner/hetzner_example.vsh @@ -0,0 +1,43 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.virt.hetzner +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.core.base +import freeflowuniverse.herolib.builder +import time + +import os + +console.print_header("Hetzner login.") + +//USE IF YOU WANT TO CONFIGURE THE HETZNER, ONLY DO THIS ONCE +//hetzner.configure("test")! + +mut cl:=hetzner.get("test")! + +for i in 0..5{ + println("test cache, first time slow then fast") + cl.servers_list()! +} + +println(cl.servers_list()!) + +mut serverinfo:= cl.server_info_get(name:"kristof2")! + +println(serverinfo) + +// cl.server_reset(name:"kristof2",wait:true)! + +// cl.server_rescue(name:"kristof2",wait:true)! + +console.print_header("SSH login") +mut b := builder.new()! +mut n := b.node_new(ipaddr: serverinfo.server_ip)! + +// n.hero_install()! +// n.hero_compile_debug()! + + +// mut ks:=cl.keys_get()! +// println(ks) + diff --git a/examples/virt/hetzner/readme.md b/examples/virt/hetzner/readme.md new file mode 100644 index 00000000..01f27e5d --- /dev/null +++ b/examples/virt/hetzner/readme.md @@ -0,0 +1,9 @@ + + +get the login passwd from: + +https://robot.hetzner.com/preferences/index + +```bash +curl -u "#ws+JdQtGCdL:..." https://robot-ws.your-server.de/server +``` \ No newline at end of file diff --git a/examples/virt/lima/lima_example.vsh b/examples/virt/lima/lima_example.vsh new file mode 100755 index 00000000..db825e66 --- /dev/null +++ b/examples/virt/lima/lima_example.vsh @@ -0,0 +1,27 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.virt.lima +import freeflowuniverse.herolib.core.texttools +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.installers.virt.lima as limainstaller +import os + +limainstaller.install()! + +mut virtmanager:=lima.new()! + +virtmanager.vm_delete_all()! + +// virtmanager.vm_new(reset:true,template:.alpine,name:'alpine',install_hero:false)! + + +//virtmanager.vm_new(reset:true,template:.arch,name:'arch',install_hero:true)! + +virtmanager.vm_new(reset:true,template:.ubuntucloud,name:'hero',install_hero:false)! +mut vm:=virtmanager.vm_get('hero')! + +println(vm) + +// vm.install_hero()! + +// console.print_debug_title("MYVM", vm.str()) \ No newline at end of file diff --git a/examples/virt/podman_buildah/buildah_example.vsh b/examples/virt/podman_buildah/buildah_example.vsh new file mode 100755 index 00000000..c5edf9f8 --- /dev/null +++ b/examples/virt/podman_buildah/buildah_example.vsh @@ -0,0 +1,35 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.virt.herocontainers +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.core.base +import freeflowuniverse.herolib.osal + +import freeflowuniverse.herolib.installers.virt.pacman + + +mut installer:= pacman.get()! + +//installer.destroy()! +//installer.install()! + +//exit(0) + +//interative means will ask for login/passwd + +mut engine:=herocontainers.new(install:true,herocompile:false)! + +engine.reset_all()! + +mut builder_gorust := engine.builder_go_rust()! + +//will build nodejs, python build & herolib, hero +//mut builder_hero := engine.builder_hero(reset:true)! + +//mut builder_web := engine.builder_heroweb(reset:true)! + + + +builder_gorust.shell()! + + diff --git a/examples/virt/podman_buildah/buildah_run.vsh b/examples/virt/podman_buildah/buildah_run.vsh new file mode 100755 index 00000000..283c9c3c --- /dev/null +++ b/examples/virt/podman_buildah/buildah_run.vsh @@ -0,0 +1,43 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.virt.herocontainers +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.core.base +// import freeflowuniverse.herolib.builder +import time + +import os + + +mut pm:=herocontainers.new(herocompile:true,install:false)! + +mut mybuildcontainer := pm.builder_get("builder_heroweb")! + +//bash & python can be executed directly in build container + +//any of the herocommands can be executed like this +mybuildcontainer.run(cmd:"installers -n heroweb",runtime:.herocmd)! + + + +// //following will execute heroscript in the buildcontainer +// mybuildcontainer.run( +// cmd:" + +// !!play.echo content:'this is just a test' + +// !!play.echo content:'this is another test' + +// ", +// runtime:.heroscript)! + +//there are also shortcuts for this + +//mybuildcontainer.hero_copy()! +//mybuildcontainer.shell()! + + +//mut b2:=pm.builder_get("builderv")! +//b2.shell()! + + diff --git a/examples/virt/podman_buildah/buildah_run_clean.vsh b/examples/virt/podman_buildah/buildah_run_clean.vsh new file mode 100755 index 00000000..202e26bb --- /dev/null +++ b/examples/virt/podman_buildah/buildah_run_clean.vsh @@ -0,0 +1,24 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.virt.herocontainers +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.core.base +// import freeflowuniverse.herolib.builder +import time + +import os + + +mut pm:=herocontainers.new(herocompile:false)! + +mut b:=pm.builder_new()! + +println(b) + +// mut mybuildcontainer := pm.builder_get("builderv")! + +// mybuildcontainer.clean()! + +// mybuildcontainer.commit('localhost/buildersmall')! + +b.shell()! diff --git a/examples/virt/podman_buildah/buildah_run_mdbook.vsh b/examples/virt/podman_buildah/buildah_run_mdbook.vsh new file mode 100755 index 00000000..8b803ae4 --- /dev/null +++ b/examples/virt/podman_buildah/buildah_run_mdbook.vsh @@ -0,0 +1,34 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import os +import flag + +import freeflowuniverse.herolib.virt.herocontainers +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.core.base +// import freeflowuniverse.herolib.builder +import time + +mut fp := flag.new_flag_parser(os.args) +fp.application('buildah mdbook example') +fp.limit_free_args(0, 0)! // comment this, if you expect arbitrary texts after the options +fp.skip_executable() +url := fp.string_opt('url', `u`, 'mdbook heroscript url')! + +additional_args := fp.finalize() or { + eprintln(err) + println(fp.usage()) + return +} + +mut pm:=herocontainers.new(herocompile:true,install:false)! + +mut mybuildcontainer := pm.builder_get("builder_heroweb")! + +// //bash & python can be executed directly in build container + +// //any of the herocommands can be executed like this +mybuildcontainer.run(cmd:"installers -n heroweb",runtime:.herocmd)! + +mybuildcontainer.run(cmd: 'hero mdbook -u ${url} -o', runtime: .bash)! + diff --git a/examples/virt/podman_buildah/readme.md b/examples/virt/podman_buildah/readme.md new file mode 100644 index 00000000..e69de29b diff --git a/examples/virt/runc/busybox.sh b/examples/virt/runc/busybox.sh new file mode 100644 index 00000000..96efe2a0 --- /dev/null +++ b/examples/virt/runc/busybox.sh @@ -0,0 +1,4 @@ + + mkdir -p cd /tmp/busyb +cd /tmp/busyb +podman export $(podman create busybox) | tar -C /tmp/busyb -xvf - \ No newline at end of file diff --git a/examples/virt/runc/install.sh b/examples/virt/runc/install.sh new file mode 100644 index 00000000..3ea6352a --- /dev/null +++ b/examples/virt/runc/install.sh @@ -0,0 +1 @@ +apt install \ No newline at end of file diff --git a/examples/virt/runc/readme.md b/examples/virt/runc/readme.md new file mode 100644 index 00000000..38d70aa6 --- /dev/null +++ b/examples/virt/runc/readme.md @@ -0,0 +1,6 @@ + + +## busybox + +- use docker, expand it into a directory + diff --git a/examples/virt/windows/cloudhypervisor.vsh b/examples/virt/windows/cloudhypervisor.vsh new file mode 100755 index 00000000..ea67d600 --- /dev/null +++ b/examples/virt/windows/cloudhypervisor.vsh @@ -0,0 +1,23 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.installers.virt.cloudhypervisor as cloudhypervisor_installer + +import freeflowuniverse.herolib.virt.cloudhypervisor +import freeflowuniverse.herolib.core.texttools +import freeflowuniverse.herolib.ui.console +import os + +mut ci:=cloudhypervisor_installer.get()! +ci.install(reset:true)! + +//mut vmm:=cloudhypervisor.new()! + +// virtmanager.vm_delete_all()! +// virtmanager.vm_new(reset:true,template:.alpine,name:'alpine',install_hero:true)! +// virtmanager.vm_new(reset:true,template:.ubuntu,name:'ubuntu',install_hero:true)! +// vmm.vm_new(reset:true,template:.arch,name:'arch',install_hero:true)! + +// mut vm:=virtmanager.vm_get('ubuntu')! +// vm.install_hero()! + +// console.print_debug_title("MYVM", vm.str()) \ No newline at end of file diff --git a/examples/virt/windows/readme.md b/examples/virt/windows/readme.md new file mode 100644 index 00000000..0ee9fb9a --- /dev/null +++ b/examples/virt/windows/readme.md @@ -0,0 +1,49 @@ +open vnc://37.27.132.46:5900 + +some info: + +https://simgunz.org/posts/2021-12-12-boot-windows-partition-from-linux-kvm/ + + +## to test rdp works + +nc -vz 37.27.132.46 3389 + + +## to install virtio drivers + +https://github.com/virtio-win/virtio-win-pkg-scripts/blob/master/README.md + +## status + +- the windows 10 works and is installed + +## to enable ssh + +Open Settings: +Press Win + I +Go to Apps > Optional features +Click "Add a feature" +Search for "OpenSSH Server" + + +```bash +Start-Service sshd +Set-Service -Name sshd -StartupType 'Automatic' +New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 +Get-Service sshd +ipconfig +``` + + +## how to shrink a disk later + +use https://learn.microsoft.com/en-us/sysinternals/downloads/sdelete + +then + +qemu-img convert -O qcow2 image.qcow2 image2.qcow2 + +or with commpression: + +qemu-img convert -O qcow2 -c image.qcow2 image2.qcow2 \ No newline at end of file diff --git a/examples/virt/windows/win_start copy.sh b/examples/virt/windows/win_start copy.sh new file mode 100755 index 00000000..b6bb2174 --- /dev/null +++ b/examples/virt/windows/win_start copy.sh @@ -0,0 +1,39 @@ +set -ex + +cd ~/Downloads + +#qemu-img create -f qcow2 windows.qcow2 50G + +qemu-system-x86_64 \ + -enable-kvm \ + -m 6000 \ + -cpu host,hv_relaxed,hv_spinlocks=0x1fff,hv_vapic,hv_time \ + -smp 4 \ + -drive file=windows.iso,media=cdrom \ + -vnc :0,password-secret=vnc-password \ + -vga std \ + -net nic,model=virtio \ + -net user \ + -net user,hostfwd=tcp::3389-:3389 \ + -object secret,id=vnc-password,data=kds007 \ + +#-drive file=windows.qcow2,format=qcow2,if=virtio \ + +#-vga qxl \ +# -monitor stdio +# -bios /usr/share/OVMF/OVMF_CODE_4M.fd \ + +# -device qxl-vga,vgamem_mb=32 \ + + +#once virtio installed +# -device virtio-vga,virgl=on,max_vram=16384 \ + +# -chardev socket,id=chrtpm,path=/tmp/swtpm-sock \ + # -tpmdev emulator,id=tpm0,chardev=chrtpm \ + # -device tpm-tis,tpmdev=tpm0 \ + +# -boot order=d \ + +#-drive if=pflash,format=raw,file=/usr/share/OVMF/OVMF_CODE_4M.fd \ + \ No newline at end of file diff --git a/examples/virt/windows/win_start_uefi_tofix.sh b/examples/virt/windows/win_start_uefi_tofix.sh new file mode 100755 index 00000000..00f9740e --- /dev/null +++ b/examples/virt/windows/win_start_uefi_tofix.sh @@ -0,0 +1,38 @@ +set -ex + +cd ~/Downloads + +swtpm socket --tpmstate dir=/tmp/mytpm1 --ctrl type=unixio,path=/tmp/swtpm-sock --log level=20 & + +qemu-system-x86_64 \ + -enable-kvm \ + -m 8000 \ + -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd \ + -drive if=pflash,format=raw,file=/usr/share/OVMF/OVMF_VARS_4M.fd \ + -cpu host,hv_relaxed,hv_spinlocks=0x1fff,hv_vapic,hv_time \ + -smp 4 \ + -boot order=dc \ + -cdrom windows.iso \ + -drive file=windows.qcow2,format=qcow2,if=virtio \ + -vnc :0,password-secret=vnc-password \ + -vga cirrus \ + -net nic,model=virtio \ + -net user \ + -net user,hostfwd=tcp::3389-:3389 \ + -chardev socket,id=chrtpm,path=/tmp/swtpm-sock \ + -tpmdev emulator,id=tpm0,chardev=chrtpm \ + -device tpm-tis,tpmdev=tpm0 \ + -object secret,id=vnc-password,data=kds007 \ + -monitor stdio + +# Block format 'qcow2' does not support the option 'bootindex + +# -drive file=windows.qcow2,format=qcow2,if=virtio \ + + +# it works without this but then we don't have the UEFI +# -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd \ +# -drive if=pflash,format=raw,file=/usr/share/OVMF/OVMF_VARS_4M.fd \ + + +#it should be /usr/share/OVMF/OVMF_CODE_4M.secboot.fd \ No newline at end of file diff --git a/examples/virt/windows/win_start_win10.sh b/examples/virt/windows/win_start_win10.sh new file mode 100755 index 00000000..4c88bb5c --- /dev/null +++ b/examples/virt/windows/win_start_win10.sh @@ -0,0 +1,33 @@ +set -ex + +cd ~/Downloads + + +qemu-system-x86_64 \ + -enable-kvm \ + -m 8000 \ + -cpu host,hv_relaxed,hv_spinlocks=0x1fff,hv_vapic,hv_time \ + -smp 4 \ + -drive file=windows.qcow2,format=qcow2 \ + -vnc :0,password-secret=vnc-password \ + -vga qxl \ + -cdrom virtio-win.iso \ + -net nic \ + -net user,hostfwd=tcp::3389-:3389,hostfwd=tcp::9922-:22 \ + -object secret,id=vnc-password,data=planetfirst007 \ + -monitor stdio + +#-drive file=windows.qcow2,format=qcow2,if=virtio \ + +# -net nic,model=virtio \ +# Block format 'qcow2' does not support the option 'bootindex + +# -drive file=windows.qcow2,format=qcow2,if=virtio \ + + +# it works without this but then we don't have the UEFI +# -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd \ +# -drive if=pflash,format=raw,file=/usr/share/OVMF/OVMF_VARS_4M.fd \ + + +#it should be /usr/share/OVMF/OVMF_CODE_4M.secboot.fd \ No newline at end of file diff --git a/examples/virt/windows/win_start_win10_production.sh b/examples/virt/windows/win_start_win10_production.sh new file mode 100755 index 00000000..a9783a57 --- /dev/null +++ b/examples/virt/windows/win_start_win10_production.sh @@ -0,0 +1,27 @@ +set -ex + +cd ~/Downloads + + + +qemu-system-x86_64 \ + -enable-kvm \ + -m 8000 \ + -cpu host,hv_relaxed,hv_spinlocks=0x1fff,hv_vapic,hv_time \ + -smp 4 \ + -drive file=windows3.qcow2,format=qcow2,if=virtio \ + -drive file=/dev/null,format=raw,if=virtio \ + -vnc :0,password-secret=vnc-password \ + -vga qxl \ + -device virtio-net-pci,netdev=n1 \ + -netdev user,id=n1,hostfwd=tcp::3389-:3389,hostfwd=tcp::9922-:22 \ + -object secret,id=vnc-password,data=planetfirst007 \ + -monitor stdio + +# -netdev user,id=n1,hostfwd=tcp::3389-:3389,hostfwd=tcp::9923-:22 \ +# -vga virtio \ + +# -device virtio-net-pci,netdev=n1 \ +# -netdev user,id=n1,hostfwd=tcp::3389-:3389,hostfwd=tcp::9922-:22 \ + + # -drive file=windows3.qcow2,format=qcow2,if=virtio \ diff --git a/examples/webdav/webdav.vsh b/examples/webdav/webdav.vsh new file mode 100755 index 00000000..a43d9f6f --- /dev/null +++ b/examples/webdav/webdav.vsh @@ -0,0 +1,29 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.webdav +import freeflowuniverse.herolib.core.pathlib +import time +import net.http +import encoding.base64 + +file_name := 'newfile.txt' +root_dir := '/tmp/webdav' + +username := "omda" +password := "password" +hashed_password := base64.encode_str('${username}:${password}') + +mut app := webdav.new_app(root_dir: root_dir, username: username, password: password) or { + eprintln('failed to create new server: ${err}') + exit(1) +} + +app.run(spawn_: true) + +time.sleep(1 * time.second) +mut p := pathlib.get_file(path: '${root_dir}/${file_name}', create: true)! +p.write('my new file')! + +mut req := http.new_request(.get, 'http://localhost:${app.server_port}/${file_name}', '') +req.add_custom_header('Authorization', 'Basic ${hashed_password}')! +req.do()! diff --git a/examples/webtools/mdbook_markdown/.gitignore b/examples/webtools/mdbook_markdown/.gitignore new file mode 100644 index 00000000..a25c71d7 --- /dev/null +++ b/examples/webtools/mdbook_markdown/.gitignore @@ -0,0 +1,6 @@ +mdbook_example +markdown_example +markdown_example0 +doctree_example +tree_scan +*.dSYM \ No newline at end of file diff --git a/examples/webtools/mdbook_markdown/content/doc.md b/examples/webtools/mdbook_markdown/content/doc.md new file mode 100644 index 00000000..fe6924c2 --- /dev/null +++ b/examples/webtools/mdbook_markdown/content/doc.md @@ -0,0 +1,10 @@ + +## The Internet Today + +![img/today_internet_.png](today_internet.png) + + +The internet today is constructed in a centralized way, we exist many times and became a product of big centralized corporations. There is also a real cyber pandemic going on; there are cyber attacks everywhere and backdoors in most of the electronic equipment we use. This architecture is unsafe, not scalable, power hungry and unable to deliver equality. Still, more than half of our planet does not have decent affordable access to the internet. + + +Blockchain technology is a huge step in the right direction but it's a shared architecture distributed over the world where data gets replicated in many locations. It is a good technology choice for use-cases like money, smart contracts, voting, digital notary services and identity management but not at all suited for building a new internet. This is also called web4. Developers still need to develop applications connecting to multiple blockchains and centralization is often not good enough. diff --git a/examples/webtools/mdbook_markdown/content/launch.md b/examples/webtools/mdbook_markdown/content/launch.md new file mode 100644 index 00000000..f74391fd --- /dev/null +++ b/examples/webtools/mdbook_markdown/content/launch.md @@ -0,0 +1,29 @@ +# this is a test + +Some normal text //somecomment at end +- [this is link](something.md) +- ![this is link2](/r/something.jpg) +Some normal text behind + +//somecomment at start + +## ts + +![this is link2](something.jpg) + +```html + + + +``` + +

para

+ + + +In Between Comment + + diff --git a/examples/webtools/mdbook_markdown/content/links.md b/examples/webtools/mdbook_markdown/content/links.md new file mode 100644 index 00000000..0bc27139 --- /dev/null +++ b/examples/webtools/mdbook_markdown/content/links.md @@ -0,0 +1,10 @@ +![](img/planet_first.png) + +## Planet First + + +[mycommunity](freeflow:community.md) + +test + +![](freeflow:freeflow_Movement.jpg) \ No newline at end of file diff --git a/examples/webtools/mdbook_markdown/content/macros.md b/examples/webtools/mdbook_markdown/content/macros.md new file mode 100644 index 00000000..952e9b91 --- /dev/null +++ b/examples/webtools/mdbook_markdown/content/macros.md @@ -0,0 +1,115 @@ +# this is a test + +Its our attempt to use markdown to execute a certain state. + +It allows us to specify parser which need to be done, only !! at start of line is executed. + +```yaml +yes = 1 +``` + +"""yaml +yes = 2 +""" + +comments allowed, the next thing is not useful but good for testing, it specifies name argument on git.ensure, its the same anyhow + +!!action.define name:download_git + +!!git.ensure url:'https://github.com/threefoldfoundation/info_foundation' autocommit:'mychanges' update:true + +!!git.ensure url:https://github.com/threefoldfoundation/legal autocommit:'mychanges' update:true + name:legal + +> can be multiline + +!!git.ensure url:https://github.com/threefoldfoundation/info_gridmanual + autocommit:'mychanges' + update:true + + + +!!publisher.server.params port:8082 + +!!action.define name:server_start depends:download_git,get_content + +!!publisher.server.start + +!!action.run name:server_start + +can be in code block or without, it does not matter + +## git + + + + +## action does not have to be in code block + +!!action.define2 name:get_content + +> if we don't specify source of content, then its the document itself + +!!markdown.section.get1 start:'### git ensure' name:testcontent.blue + +> we now remove the first line which matched, the last line is never included, but now we forced to include it + 1 extra line + +!!markdown.section.get2 start:'### git ensure' end:'# end of' name:testcontent.red trim_end:+3 trim_start:+1 + +> fetch a full document, there will be no start/stop + +!!markdown.section.get3 name:testcontent.pink gitrepo:info_foundation + file:'communication.md' + +### end of block + +This should be in the found block. + +### we can even do task management with it + + +!!taskcontext name:myproject1 + +!!task id:a1 + name:'need to do something 1' + description: + ## markdown works in it + + description can be multiline + lets see what happens + + - a + - something else + + ### subtitle + + ```python + #even code block in the other block, crazy parsing for sure + def test(): + print("test") + ``` + priority:10 + + +//lets now create another task which depends on the previous one + +!!task id:a1 name:'something else' assign:'kristof(50%),jan' effort:1d + depends:a1 + +!!dao.deposit currency:usdc amount:10 +!!dao.withdraw currency:usdc amount:10 + +!!dao.pool.deposit currency:usdc amount:10 +!!dao.pool.withdraw currency:usdc amount:10 + +!!dao.pool.buy currency:tft amount:100 + +!!dao.wallets.info currency:usdc +!!dao.wallets.info + +!!dao.exchange.buy currency:tft amount:100 price_max_usd:0.05 deadline:10h + diff --git a/examples/webtools/mdbook_markdown/content/test.md b/examples/webtools/mdbook_markdown/content/test.md new file mode 100644 index 00000000..00b22b0e --- /dev/null +++ b/examples/webtools/mdbook_markdown/content/test.md @@ -0,0 +1,51 @@ +# this is a header +## another header + +This is some text. +Same paragraph [my link](https://github.com/vlang/v/blob/master/doc/docs.md). + +Next paragraph. + +- list item + - list item 2 + - list item 3 +- list item 4 + + +!!another.action key:val key2:val2 +!!actor.name k1:v1 + k2:v2 + +# header2 +text in paragraph + +| Month | Savings | +| -------- | ------- | +| January | $250 | +| February | $80 | +| March | $420 | + +```vlang + fn main(){ + println('hello world') + } +``` + +```js +!!another.action2 key:val key2:val2 +!!myactor.myname k1:v1 + k2:v2 + +``` + +!!include this is an include + +text on same line as beginning tag is ignored +this text is part of the html element +# this header too +``` + codeblock too +``` +this is ignored too + + \ No newline at end of file diff --git a/examples/webtools/mdbook_markdown/content/test_para.md b/examples/webtools/mdbook_markdown/content/test_para.md new file mode 100644 index 00000000..352263f2 --- /dev/null +++ b/examples/webtools/mdbook_markdown/content/test_para.md @@ -0,0 +1,9 @@ + +qwpoejfpqwoejf +qoweifjjf + +- list item + - list item 2 + - list item 3 +- list item 4 +test diff --git a/examples/webtools/mdbook_markdown/doctree_export.vsh b/examples/webtools/mdbook_markdown/doctree_export.vsh new file mode 100755 index 00000000..c683defd --- /dev/null +++ b/examples/webtools/mdbook_markdown/doctree_export.vsh @@ -0,0 +1,27 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.herolib.data.doctree + +mut tree := doctree.new(name: 'test')! + +// path string +// heal bool = true // healing means we fix images +// git_url string +// git_reset bool +// git_root string +// git_pull bool +// load bool = true // means we scan automatically the added collection +for project in 'projectinca, legal, why, web4,tfgrid3'.split(',').map(it.trim_space()) { + tree.scan( + git_url: 'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/development/collections/${project}' + git_pull: false + )! +} + +tree.export( + dest: '/tmp/test' + reset: true + keep_structure: true + exclude_errors: false + production: true +)! diff --git a/examples/webtools/mdbook_markdown/markdown_example.vsh b/examples/webtools/mdbook_markdown/markdown_example.vsh new file mode 100755 index 00000000..9611efed --- /dev/null +++ b/examples/webtools/mdbook_markdown/markdown_example.vsh @@ -0,0 +1,44 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +// import freeflowuniverse.herolib.core.texttools +import freeflowuniverse.herolib.ui.console +import log +import freeflowuniverse.herolib.data.markdownparser +import os + +console.print_header('Print markdown in treeview.') +println('') +// mut session := play.session_new( +// context_name: "test" +// interactive: true +// )! + +// get path local to the current script +path_my_content := '${os.dir(@FILE)}/content' + +mut doc1 := markdownparser.new( + path: '${path_my_content}/test.md' +)! +content1 := doc1.markdown()! + +println(doc1.treeview()) + +console.print_header('Markdown output:') +println('') +println(content1) + +console.print_header('Actions:') +println('') +println(doc1.actions(actor: 'myactor', name: 'myname')) + +// console.print_header('Action Pointers:') +// println('') +// for a in doc1.action_pointers(actor: 'myactor', name: 'myname') { +// doc1.content_set(a.element_id, '> THIS IS WHAT WE FILL IN FROM ACTOR') +// println(a) +// } + +console.print_header("Markdown output after macro's:") +println('') +content2 := doc1.treeview() +println(content2) diff --git a/lib/builder/bootstrapper.v b/lib/builder/bootstrapper.v new file mode 100644 index 00000000..9821deae --- /dev/null +++ b/lib/builder/bootstrapper.v @@ -0,0 +1,270 @@ +module builder + +import os +import freeflowuniverse.herolib.core.texttools +import freeflowuniverse.herolib.core.pathlib +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.ui +import v.embed_file + +const heropath_ = os.dir(@FILE) + '/../' + +pub struct BootStrapper { +pub mut: + embedded_files map[string]embed_file.EmbedFileData @[skip; str: skip] +} + +@[params] +pub struct BootstrapperArgs { +pub mut: + name string = 'bootstrap' + addr string // format: root@something:33, 192.168.7.7:222, 192.168.7.7, despiegk@something + reset bool + debug bool +} + +fn (mut bs BootStrapper) load() { + bs.embedded_files['install_base.sh'] = $embed_file('../../scripts/install_base.sh') + bs.embedded_files['install_hero.sh'] = $embed_file('../../scripts/install_hero.sh') +} + +// to use do something like: export NODES="195.192.213.3" . +pub fn bootstrapper() BootStrapper { + mut bs := BootStrapper{} + bs.load() + return bs +} + +pub fn (mut bs BootStrapper) run(args_ BootstrapperArgs) ! { + mut args := args_ + addr := texttools.to_array(args.addr) + mut b := new()! + mut counter := 0 + for a in addr { + counter += 1 + name := '${args.name}_${counter}' + mut n := b.node_new(ipaddr: a, name: name)! + n.hero_install()! + n.hero_install()! + } +} + +pub fn (mut node Node) upgrade() ! { + mut bs := bootstrapper() + install_base_content_ := bs.embedded_files['install_base.sh'] or { panic('bug') } + install_base_content := install_base_content_.to_string() + cmd := '${install_base_content}\n' + node.exec_cmd( + cmd: cmd + period: 48 * 3600 + reset: false + description: 'upgrade operating system packages' + )! +} + +pub fn (mut node Node) hero_install() ! { + console.print_debug('install hero') + mut bs := bootstrapper() + install_hero_content_ := bs.embedded_files['install_hero.sh'] or { panic('bug') } + install_hero_content := install_hero_content_.to_string() + if node.platform == .osx { + // we have no choice then to do it interactive + myenv := node.environ_get()! + homedir := myenv['HOME'] or { return error("can't find HOME in env") } + node.exec_silent('mkdir -p ${homedir}/hero/bin')! + node.file_write('${homedir}/hero/bin/install.sh', install_hero_content)! + node.exec_silent('chmod +x ${homedir}/hero/bin/install.sh')! + node.exec_interactive('${homedir}/hero/bin/install.sh')! + } else if node.platform == .ubuntu { + myenv := node.environ_get()! + homedir := myenv['HOME'] or { return error("can't find HOME in env") } + node.exec_silent('mkdir -p ${homedir}/hero/bin')! + node.file_write('${homedir}/hero/bin/install.sh', install_hero_content)! + node.exec_silent('chmod +x ${homedir}/hero/bin/install.sh')! + node.exec_interactive('${homedir}/hero/bin/install.sh')! + } +} + +pub fn (mut node Node) dagu_install() ! { + console.print_debug('install dagu') + if !osal.cmd_exists('dagu') { + _ = bootstrapper() + node.exec_silent('curl -L https://raw.githubusercontent.com/yohamta/dagu/main/scripts/downloader.sh | bash')! + } +} + +@[params] +pub struct HeroInstallArgs { +pub mut: + reset bool +} + +pub fn (mut node Node) hero_install(args HeroInstallArgs) ! { + mut bs := bootstrapper() + install_base_content_ := bs.embedded_files['install_base.sh'] or { panic('bug') } + install_base_content := install_base_content_.to_string() + + if args.reset { + console.clear() + console.print_debug('') + console.print_stderr('will remove: .vmodules, hero lib code and ~/hero') + console.print_debug('') + mut myui := ui.new()! + toinstall := myui.ask_yesno( + question: 'Ok to reset?' + default: true + )! + if !toinstall { + exit(1) + } + os.rmdir_all('${os.home_dir()}/.vmodules')! + os.rmdir_all('${os.home_dir()}/hero')! + os.rmdir_all('${os.home_dir()}/code/github/freeflowuniverse/herolib')! + os.rmdir_all('${os.home_dir()}/code/github/freeflowuniverse/webcomponents')! + } + + cmd := ' + ${install_base_content} + + rm -f /usr/local/bin/hero + freeflow_dev_env_install + + ~/code/github/freeflowuniverse/herolib/install.sh + + echo HERO, V, CRYSTAL ALL INSTALL OK + echo WE ARE READY TO HERO... + + ' + console.print_debug('executing cmd ${cmd}') + node.exec_cmd(cmd: cmd)! +} + +@[params] +pub struct HeroUpdateArgs { +pub mut: + sync_from_local bool // will sync local hero lib to the remote, then cannot use git + sync_full bool // sync the full herolib repo + sync_fast bool = true // don't hash the files, there is small chance on error + git_reset bool // will get the code from github at remote and reset changes + git_pull bool // will pull the code but not reset, will give error if it can't reset + branch string +} + +// execute vscript on remote node + +pub fn (mut node Node) hero_update(args_ HeroUpdateArgs) ! { + mut args := args_ + + if args.sync_from_local && (args.git_reset || args.git_pull) { + return error('if sync is asked for hero update, then cannot use git to get hero') + } + + if args.sync_from_local { + if args.sync_full { + node.sync_code('hero', builder.heropath_ + '/..', '~/code/github/freeflowuniverse/herolib', + args.sync_fast)! + } else { + node.sync_code('hero_lib', builder.heropath_, '~/code/github/freeflowuniverse/herolib/herolib', + args.sync_fast)! + } + return + } + mut branchstr := '' + if args.branch.len > 0 { + branchstr = 'git checkout ${args.branch} -f && git pull' // not sure latest git pull needed + } + if args.git_reset { + args.git_pull = false + node.exec_cmd( + cmd: ' + cd ~/code/github/freeflowuniverse/herolib + rm -f .git/index + git fetch --all + git reset HEAD --hard + git clean -xfd + git checkout . -f + ${branchstr} + ' + )! + } + if args.git_pull { + node.exec_cmd( + cmd: ' + cd ~/code/github/freeflowuniverse/herolib + git pull + ${branchstr} + ' + )! + } +} + +pub fn (mut node Node) sync_code(name string, src_ string, dest string, fast_rsync bool) ! { + mut src := pathlib.get_dir(path: os.abs_path(src_), create: false)! + node.upload( + source: src.path + dest: dest + ignore: [ + '.git/*', + '_archive', + '.vscode', + 'examples', + ] + delete: true + fast_rsync: fast_rsync + )! +} + +// sync local hero code to rmote and then compile hero +pub fn (mut node Node) hero_compile_sync() ! { + if !node.file_exists('~/code/github/freeflowuniverse/herolib/cli/readme.md') { + node.hero_install()! + } + node.hero_update()! + node.exec_cmd( + cmd: ' + ~/code/github/freeflowuniverse/herolib/install.sh + ~/code/github/freeflowuniverse/herolib/cli/hero/compile_debug.sh + ' + )! +} + +pub fn (mut node Node) hero_compile() ! { + if !node.file_exists('~/code/github/freeflowuniverse/herolib/cli/readme.md') { + node.hero_install()! + } + node.exec_cmd( + cmd: ' + ~/code/github/freeflowuniverse/herolib/cli/hero/compile_debug.sh + ' + )! +} + +@[params] +pub struct VScriptArgs { +pub mut: + path string + sync_from_local bool // will sync local hero lib to the remote + git_reset bool // will get the code from github at remote and reset changes + git_pull bool // will pull the code but not reset, will give error if it can't reset + branch string // can only be used when git used +} + +pub fn (mut node Node) vscript(args_ VScriptArgs) ! { + mut args := args_ + node.hero_update( + sync_from_local: args.sync_from_local + git_reset: args.git_reset + git_pull: args.git_pull + branch: args.branch + )! + mut p := pathlib.get_file(path: args.path, create: false)! + + node.upload(source: p.path, dest: '/tmp/remote_${p.name()}')! + node.exec_cmd( + cmd: ' + cd /tmp/remote + v -w -enable-globals /tmp/remote_${p.name()} + ' + )! +} diff --git a/lib/builder/builder_factory.v b/lib/builder/builder_factory.v new file mode 100644 index 00000000..bd2d2a57 --- /dev/null +++ b/lib/builder/builder_factory.v @@ -0,0 +1,24 @@ +module builder + +import freeflowuniverse.herolib.core.base + +@[heap] +pub struct BuilderFactory { +} + +pub fn new() !BuilderFactory { + mut c := base.context()! + mut bf := BuilderFactory{} + return bf +} + +@[params] +pub struct NodeLocalArgs { +pub: + reload bool +} + +pub fn node_local(args NodeLocalArgs) !&Node { + mut bldr := new()! + return bldr.node_new(name: 'localhost', reload: args.reload) +} diff --git a/lib/builder/done.v b/lib/builder/done.v new file mode 100644 index 00000000..fbb6c1d4 --- /dev/null +++ b/lib/builder/done.v @@ -0,0 +1,52 @@ +module builder + +import freeflowuniverse.herolib.ui.console + +pub fn (mut node Node) done_set(key string, val string) ! { + if key in node.done { + if node.done[key] == val { + return + } + } + node.done[key] = val + node.save()! +} + +pub fn (mut node Node) done_get(key string) ?string { + if key !in node.done { + return none + } + return node.done[key] +} + +// will return empty string if it doesnt exist +pub fn (mut node Node) done_get_str(key string) string { + if key !in node.done { + return '' + } + return node.done[key] +} + +// will return 0 if it doesnt exist +pub fn (mut node Node) done_get_int(key string) int { + if key !in node.done { + return 0 + } + return node.done[key].int() +} + +pub fn (mut node Node) done_exists(key string) bool { + return key in node.done +} + +pub fn (mut node Node) done_print() { + console.print_header('DONE: ${node.name} ') + for key, val in node.done { + console.print_item('${key} = ${val}') + } +} + +pub fn (mut node Node) done_reset() ! { + node.done = map[string]string{} + node.save()! +} diff --git a/lib/builder/executor.v b/lib/builder/executor.v new file mode 100644 index 00000000..13d91ce3 --- /dev/null +++ b/lib/builder/executor.v @@ -0,0 +1,58 @@ +module builder + +import freeflowuniverse.herolib.data.ipaddress + +type Executor = ExecutorLocal | ExecutorSSH + +pub struct ExecutorNewArguments { +pub mut: + local bool // if this set then will always be the local machine + ipaddr string + user string = 'root' + debug bool +} + +// create new executor (is way how to execute in std way onto a local or remote machine) +// pub struct ExecutorNewArguments { +// local false //if this set then will always be the local machine +// ipaddr string +// user string = "root" +// debug bool +// } +//- format ipaddr: 192.168.6.6:7777 +//- format ipaddr: 192.168.6.6 +//- format ipadd6: [666:555:555:...] +//- format ipaddr: any ipv6 addr +//- if ipaddr is empty or starts with localhost or 127.0.0.1 -> will be the ExecutorLocal +fn executor_new(args_ ExecutorNewArguments) !Executor { + mut args := args_ + hasport := args.ipaddr.contains(':') + if !hasport { + args.ipaddr = args.ipaddr + ':22' + } + if args.ipaddr == '' + || (args.ipaddr.starts_with('localhost') && hasport == false) + || (args.ipaddr.starts_with('127.0.0.1') && hasport == false) { + return ExecutorLocal{ + debug: args.debug + } + } else { + ipaddr := ipaddress.new(args.ipaddr) or { + return error('can not initialize ip address.\n ${err}') + } + mut e := ExecutorSSH{ + ipaddr: ipaddr + user: args.user + debug: args.debug + } + e.init()! + return e + } +} + +@[params] +pub struct ExecArgs { +pub mut: + cmd string + stdout bool = true +} diff --git a/lib/builder/executor_local.v b/lib/builder/executor_local.v new file mode 100644 index 00000000..f8583b08 --- /dev/null +++ b/lib/builder/executor_local.v @@ -0,0 +1,118 @@ +module builder + +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.osal.rsync +// import freeflowuniverse.herolib.core.pathlib +import os + +@[heap] +pub struct ExecutorLocal { + retry int = 1 // nr of times something will be retried before failing, need to check also what error is, only things which should be retried need to be done, default 1 because is local +pub mut: + debug bool +} + +pub fn (mut executor ExecutorLocal) exec(args ExecArgs) !string { + res := osal.exec(cmd: args.cmd, stdout: args.stdout, debug: executor.debug)! + return res.output +} + +pub fn (mut executor ExecutorLocal) exec_interactive(args ExecArgs) ! { + osal.execute_interactive(args.cmd)! +} + +pub fn (mut executor ExecutorLocal) file_write(path string, text string) ! { + // console.print_debug('local write ${path}') + return os.write_file(path, text) +} + +pub fn (mut executor ExecutorLocal) file_read(path string) !string { + return os.read_file(path) +} + +pub fn (mut executor ExecutorLocal) file_exists(path string) bool { + return os.exists(path) +} + +pub fn (mut executor ExecutorLocal) debug_on() { + executor.debug = true +} + +pub fn (mut executor ExecutorLocal) debug_off() { + executor.debug = false +} + +// carefull removes everything +pub fn (mut executor ExecutorLocal) delete(path string) ! { + if os.is_file(path) || os.is_link(path) { + return os.rm(path) + } else if os.is_dir(path) { + return os.rmdir_all(path) + } + return +} + +// get environment variables from the executor +pub fn (mut executor ExecutorLocal) environ_get() !map[string]string { + env := os.environ() + if false { + return error('can never happen') + } + return env +} + +/* +Executor info or meta data +accessing type Executor won't allow to access the +fields of the struct, so this is workaround +*/ +pub fn (mut executor ExecutorLocal) info() map[string]string { + return { + 'category': 'local' + } +} + +// upload from local FS to executor FS +pub fn (mut executor ExecutorLocal) upload(args SyncArgs) ! { + mut rsargs := rsync.RsyncArgs{ + source: args.source + dest: args.dest + delete: args.delete + ignore: args.ignore + ignore_default: args.ignore_default + stdout: args.stdout + } + rsync.rsync(rsargs)! +} + +// download from executor FS to local FS +pub fn (mut executor ExecutorLocal) download(args SyncArgs) ! { + mut rsargs := rsync.RsyncArgs{ + source: args.source + dest: args.dest + delete: args.delete + ignore: args.ignore + ignore_default: args.ignore_default + stdout: args.stdout + } + rsync.rsync(rsargs)! +} + +pub fn (mut executor ExecutorLocal) shell(cmd string) ! { + if cmd.len > 0 { + os.execvp('/bin/bash', ["-c '${cmd}'"])! + } else { + os.execvp('/bin/bash', [])! + } +} + +pub fn (mut executor ExecutorLocal) list(path string) ![]string { + if !executor.dir_exists(path) { + panic('Dir Not found') + } + return os.ls(path) +} + +pub fn (mut executor ExecutorLocal) dir_exists(path string) bool { + return os.is_dir(path) +} diff --git a/lib/builder/executor_local_test.v b/lib/builder/executor_local_test.v new file mode 100644 index 00000000..6c17cc0e --- /dev/null +++ b/lib/builder/executor_local_test.v @@ -0,0 +1,36 @@ +module builder + +import freeflowuniverse.herolib.ui.console + +fn test_exec() { + mut e := ExecutorLocal{} + res := e.exec(cmd: 'ls /') or { panic('error execution') } + console.print_debug(res) +} + +fn test_file_operations() { + mut e := ExecutorLocal{} + e.file_write('/tmp/abc.txt', 'abc') or { panic('can not write file') } + mut text := e.file_read('/tmp/abc.txt') or { panic('can not read file') } + assert text == 'abc' + mut exists := e.file_exists('/tmp/abc.txt') + assert exists == true + e.delete('/tmp/abc.txt') or { panic(err) } + exists = e.file_exists('/tmp/abc.txt') + assert exists == false +} + +fn test_environ_get() { + mut e := ExecutorLocal{} + mut env := e.environ_get() or { panic(err) } + console.print_debug('${env}') +} + +// TODO: fix +// fn test_node_new() { +// mut factory := new()! +// mut node := factory.node_new(name: 'localhost', reload: true) or { +// panic("Can't get new node: ${err}") +// } +// console.print_debug('${node}') +// } diff --git a/lib/builder/executor_ssh.v b/lib/builder/executor_ssh.v new file mode 100644 index 00000000..66b4b7cd --- /dev/null +++ b/lib/builder/executor_ssh.v @@ -0,0 +1,255 @@ +module builder + +import os +import rand +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.osal.rsync +import freeflowuniverse.herolib.core.pathlib +import freeflowuniverse.herolib.data.ipaddress +import freeflowuniverse.herolib.ui.console + +@[heap] +pub struct ExecutorSSH { +pub mut: + ipaddr ipaddress.IPAddress + sshkey string + user string = 'root' // default will be root + initialized bool + retry int = 1 // nr of times something will be retried before failing, need to check also what error is, only things which should be retried need to be done + debug bool = true +} + +fn (mut executor ExecutorSSH) init() ! { + if !executor.initialized { + // if executor.ipaddr.port == 0 { + // return error('port cannot be 0.\n${executor}') + // } + // TODO: need to call code from SSHAGENT do not reimplement here, not nicely done + os.execute('pgrep -x ssh-agent || eval `ssh-agent -s`') + + if executor.sshkey != '' { + osal.exec(cmd: 'ssh-add ${executor.sshkey}')! + } + mut addr := executor.ipaddr.addr + if addr == '' { + addr = 'localhost' + } + if executor.ipaddr.port == 0 { + executor.ipaddr.port = 22 + } + // TODO: doesn't work with ipv6 after working with ipv4, need better check too, because this slows everything down + // cmd := "sh -c 'ssh-keyscan -H ${executor.ipaddr.addr} -p ${executor.ipaddr.port} -t ecdsa-sha2-nistp256 2>/dev/null >> ~/.ssh/known_hosts'" + // osal.execute_silent(cmd) or { return error('cannot add the ssh keys to known hosts') } + executor.initialized = true + } +} + +pub fn (mut executor ExecutorSSH) debug_on() { + executor.debug = true +} + +pub fn (mut executor ExecutorSSH) debug_off() { + executor.debug = false +} + +pub fn (mut executor ExecutorSSH) exec(args_ ExecArgs) !string { + mut args := args_ + if executor.debug { + console.print_debug('execute ${executor.ipaddr.addr}: ${args.cmd}') + } + mut port := '' + if executor.ipaddr.port > 10 { + port = '-p ${executor.ipaddr.port}' + } + args.cmd = 'ssh -o StrictHostKeyChecking=no ${executor.user}@${executor.ipaddr.addr} ${port} "${args.cmd}"' + res := osal.exec(cmd: args.cmd, stdout: args.stdout, debug: executor.debug)! + return res.output +} + +pub fn (mut executor ExecutorSSH) exec_interactive(args_ ExecArgs) ! { + mut args := args_ + mut port := '' + if executor.ipaddr.port > 10 { + port = '-p ${executor.ipaddr.port}' + } + args.cmd = 'ssh -tt -o StrictHostKeyChecking=no ${executor.user}@${executor.ipaddr.addr} ${port} "${args.cmd}"' + osal.execute_interactive(args.cmd)! +} + +pub fn (mut executor ExecutorSSH) file_write(path string, text string) ! { + if executor.debug { + console.print_debug('${executor.ipaddr.addr} file write: ${path}') + } + local_path := '/tmp/${rand.uuid_v4()}' + os.write_file(local_path, text)! + executor.upload(source: local_path, dest: path, stdout: false)! + os.rm(local_path)! +} + +pub fn (mut executor ExecutorSSH) file_read(path string) !string { + if executor.debug { + console.print_debug('${executor.ipaddr.addr} file read: ${path}') + } + local_path := '/tmp/${rand.uuid_v4()}' + executor.download(source: path, dest: local_path)! + r := os.read_file(local_path)! + os.rm(local_path) or { panic(err) } + return r +} + +pub fn (mut executor ExecutorSSH) file_exists(path string) bool { + if executor.debug { + console.print_debug('${executor.ipaddr.addr} file exists: ${path}') + } + output := executor.exec(cmd: 'test -f ${path} && echo found || echo not found', stdout: false) or { + return false + } + if output == 'found' { + return true + } + return false +} + +// carefull removes everything +pub fn (mut executor ExecutorSSH) delete(path string) ! { + if executor.debug { + console.print_debug('${executor.ipaddr.addr} file delete: ${path}') + } + executor.exec(cmd: 'rm -rf ${path}', stdout: false) or { panic(err) } +} + +// upload from local FS to executor FS +pub fn (mut executor ExecutorSSH) download(args SyncArgs) ! { + mut addr := '${executor.user}@${executor.ipaddr.addr}:${executor.ipaddr.port}' + if executor.ipaddr.cat == .ipv6 { + addr = '\'${executor.user}@[${executor.ipaddr.addr}]\':${executor.ipaddr.port}' + } + mut rsargs := rsync.RsyncArgs{ + source: args.source + dest: args.dest + delete: args.delete + ipaddr_src: addr + ignore: args.ignore + ignore_default: args.ignore_default + stdout: args.stdout + } + rsync.rsync(rsargs)! +} + +// download from executor FS to local FS +pub fn (mut executor ExecutorSSH) upload(args SyncArgs) ! { + mut p := pathlib.get(args.source) + if !p.exists() { + return error('Cannot upload ${args}') + } + + mut psize := p.size_kb()! + + // source string + // dest string + // delete bool // do we want to delete the destination + // ipaddr string // e.g. root@192.168.5.5:33 (can be without root@ or :port) + // ignore []string // arguments to ignore e.g. ['*.pyc','*.bak'] + // ignore_default bool = true // if set will ignore a common set + // stdout bool = true + // fast_rsync bool = true + if args.ignore.len == 0 && psize < 100 { + mut addr2 := '${executor.user}@${executor.ipaddr.addr}:${args.dest}' + if executor.ipaddr.cat == .ipv6 { + addr2 = '\'${executor.user}@[${executor.ipaddr.addr}]\':${args.dest}' + } + cmd := "scp -o \"StrictHostKeyChecking=no\" -P ${executor.ipaddr.port} ${args.source} ${addr2}" + // console.print_debug(cmd) + res := os.execute(cmd) + if res.exit_code > 0 { + return error('cannot upload over ssh: ${cmd}') + } + return + } + + mut addr := '${executor.user}@${executor.ipaddr.addr}:${executor.ipaddr.port}' + if executor.ipaddr.cat == .ipv6 { + addr = '\'${executor.user}@[${executor.ipaddr.addr}]\':${executor.ipaddr.port}' + } + + mut rsargs := rsync.RsyncArgs{ + source: args.source + dest: args.dest + delete: args.delete + ipaddr_dst: addr + ignore: args.ignore + ignore_default: args.ignore_default + stdout: args.stdout + fast_rsync: args.fast_rsync + } + rsync.rsync(rsargs)! +} + +// get environment variables from the executor +pub fn (mut executor ExecutorSSH) environ_get() !map[string]string { + env := executor.exec(cmd: 'env', stdout: false) or { return error('can not get environment') } + // if executor.debug { + // console.print_header(' ${executor.ipaddr.addr} env get') + // } + + mut res := map[string]string{} + if env.contains('\n') { + for line in env.split('\n') { + if line.contains('=') { + splitted := line.split('=') + key := splitted[0].trim(' ') + val := splitted[1].trim(' ') + res[key] = val + } + } + } + return res +} + +/* +Executor info or meta data +accessing type Executor won't allow to access the +fields of the struct, so this is workaround +*/ +pub fn (mut executor ExecutorSSH) info() map[string]string { + return { + 'category': 'ssh' + 'sshkey': executor.sshkey + 'user': executor.user + 'ipaddress': executor.ipaddr.addr + 'port': '${executor.ipaddr.port}' + } +} + +// ssh shell on the node default ssh port, or any custom port that may be +// forwarding ssh traffic to certain container + +pub fn (mut executor ExecutorSSH) shell(cmd string) ! { + if cmd.len > 0 { + panic('TODO IMPLEMENT SHELL EXEC OVER SSH') + } + os.execvp('ssh', ['-o StrictHostKeyChecking=no', '${executor.user}@${executor.ipaddr.addr}', + '-p ${executor.ipaddr.port}'])! +} + +pub fn (mut executor ExecutorSSH) list(path string) ![]string { + if !executor.dir_exists(path) { + panic('Dir Not found') + } + mut res := []string{} + output := executor.exec(cmd: 'ls ${path}', stdout: false)! + for line in output.split('\n') { + res << line + } + return res +} + +pub fn (mut executor ExecutorSSH) dir_exists(path string) bool { + output := executor.exec(cmd: 'test -d ${path} && echo found || echo not found', stdout: false) or { + return false + } + if output.trim_space() == 'found' { + return true + } + return false +} diff --git a/lib/builder/executor_ssh_test.v b/lib/builder/executor_ssh_test.v new file mode 100644 index 00000000..20f8861e --- /dev/null +++ b/lib/builder/executor_ssh_test.v @@ -0,0 +1,78 @@ +module builder + +import rand +import freeflowuniverse.herolib.data.ipaddress { IPAddress } +import freeflowuniverse.herolib.ui.console + +// Assuming this function runs first (which is the case) +// This generates ssh keys on local machine to use for +// connecting to local host over ssh and test executor +fn testsuite_begin() { + mut e := ExecutorLocal{} + e.exec(cmd: "yes | ssh-keygen -t rsa -f ~/.ssh/id_rsa_test -N ''")! + e.exec(cmd: 'chmod 0600 ~/.ssh/id_rsa_test && chmod 0644 ~/.ssh/id_rsa_test.pub')! + e.exec(cmd: 'cat ~/.ssh/id_rsa_test.pub >> ~/.ssh/authorized_keys')! + e.exec(cmd: 'chmod og-wx ~/.ssh/authorized_keys')! +} + +fn test_exec() { + mut e := ExecutorSSH{ + sshkey: '~/.ssh/id_rsa_test' + } + e.ipaddr = IPAddress{ + addr: '127.0.0.1' + port: 22 + } + res := e.exec(cmd: 'ls /')! + console.print_debug(res) +} + +fn test_file_operations() { + mut e := ExecutorSSH{ + sshkey: '~/.ssh/id_rsa_test' + } + e.ipaddr = IPAddress{ + addr: '127.0.0.1' + port: 22 + cat: .ipv4 + } + mut filepath := '/tmp/${rand.uuid_v4()}' + e.file_write(filepath, 'ssh')! + mut text := e.file_read(filepath)! + assert text == 'ssh' + mut exists := e.file_exists(filepath) + assert exists == true + e.delete(filepath)! + exists = e.file_exists(filepath) + assert exists == false +} + +fn test_environ_get() { + mut e := ExecutorSSH{ + sshkey: '~/.ssh/id_rsa_test' + } + e.ipaddr = IPAddress{ + addr: '127.0.0.1' + port: 22 + cat: .ipv4 + } + mut env := e.environ_get()! + console.print_debug(env) +} + +// fn test_remote_machine() { +// mut e := ExecutorSSH { +// sshkey: "~/.ssh/id_rsa_test", +// user: "root", +// ipaddr: IPAddress{ +// addr: "104.236.53.191", +// port: Port{ +// number: 22, +// cat: PortType.tcp +// }, +// cat: IpAddressType.ipv4 +// } +// } +// res := e.exec(cmd: "ls /root")! +// console.print_debug(res) +// } diff --git a/lib/builder/model_package.v b/lib/builder/model_package.v new file mode 100644 index 00000000..b4a9292c --- /dev/null +++ b/lib/builder/model_package.v @@ -0,0 +1,39 @@ +module builder + +// is e.g. an ubuntu packagedapp, it needs to be packaged by the package maintainers ! +pub struct Package { + name string + description string + version string + aliases []PackageAlias +} + +// if there is an exception of how package needs to be installed (alias) +// e.g. on ubuntu something is called myapp but on alpine its my_app +pub struct PackageAlias { + name string + platformtype PlatformType + version string +} + +// get the right name depending the platform type +pub fn (mut package Package) name_get(platformtype PlatformType) string { + for alias in package.aliases { + if alias.platformtype == platformtype { + return alias.name + } + } + return package.name +} + +// get the right name depending the platform type +pub fn (mut package Package) version_get(platformtype PlatformType) string { + for alias in package.aliases { + if alias.platformtype == platformtype { + if alias.version != '' { + return alias.version + } + } + } + return package.version +} diff --git a/lib/builder/node.v b/lib/builder/node.v new file mode 100644 index 00000000..81f27526 --- /dev/null +++ b/lib/builder/node.v @@ -0,0 +1,97 @@ +module builder + +import json +import freeflowuniverse.herolib.data.paramsparser { Params } +import freeflowuniverse.herolib.core.base +// import freeflowuniverse.herolib.ui.console +import crypto.md5 + +pub enum PlatformType { + unknown + osx + ubuntu + alpine + arch +} + +pub enum CPUType { + unknown + intel + arm + intel32 + arm32 +} + +@[heap] +pub struct Node { +mut: + factory &BuilderFactory @[skip; str: skip] +pub mut: + name string = 'unknown' + executor Executor @[skip; str: skip] + platform PlatformType + cputype CPUType + done map[string]string + environment map[string]string + params Params + hostname string +} + +// get unique key for the node, as used in caching environment +pub fn (mut node Node) key() string { + mut myhash := 'local' + if mut node.executor is ExecutorSSH { + myaddr := node.executor.ipaddr.address() or { panic(err) } + myhash = md5.hexhash(myaddr) + } + return node.name + '_' + myhash +} + +// get remote environment arguments in memory +pub fn (mut node Node) readfromsystem() ! { + node.platform_load()! + node.environment = node.environ_get(reload: true) or { + return error('can not load env.\n ${err}') + } + + home := node.environment['HOME'] or { + return error('could not find HOME in environment variables.\n ${node}') + } + node.environment['HOME'] = home.trim_space() + if node.environment['HOME'] == '' { + return error('HOME env cannot be empty for ${node.name}') + } + node.save()! +} + +// load the node from redis cache, if not there will load from system . +// return true if the data was in redis (cache) +pub fn (mut node Node) load() !bool { + mut mycontext := base.context()! + mut r := mycontext.redis()! + data := r.hget('nodes', node.key())! + if data == '' { + node.readfromsystem()! + return false + } + data2 := json.decode(Node, data)! + + node.platform = data2.platform + node.cputype = data2.cputype + node.done = data2.done.clone() + node.environment = data2.environment.clone() + node.params = data2.params + if node.environment.len == 0 { + node.readfromsystem()! + node.save()! + } + return true +} + +// get remote environment arguments in memory +pub fn (mut node Node) save() ! { + data := json.encode(node) + mut mycontext := base.context()! + mut r := mycontext.redis()! + r.hset('nodes', node.key(), data)! +} diff --git a/lib/builder/node_commands.v b/lib/builder/node_commands.v new file mode 100644 index 00000000..741fdcc9 --- /dev/null +++ b/lib/builder/node_commands.v @@ -0,0 +1,227 @@ +module builder + +import freeflowuniverse.herolib.core.texttools +import crypto.md5 +import time +import freeflowuniverse.herolib.data.ourtime +import freeflowuniverse.herolib.ui.console +// import freeflowuniverse.herolib.osal + +// check command exists on the platform, knows how to deal with different platforms +pub fn (mut node Node) cmd_exists(cmd string) bool { + return node.exec_ok('which ${cmd}') +} + +pub struct NodeExecCmd { +pub mut: + name string = 'default' + cmd string + period int // period in which we check when this was done last, if 0 then period is indefinite + reset bool = true // means do again or not + remove_installer bool = true // delete the installer + description string + stdout bool = true + checkkey string // if used will use this one in stead of hash of cmd, to check if was executed already + tmpdir string + ignore_error_codes []int +} + +// return the ipaddress as known on the public side +// is using resolver4.opendns.com +pub fn (mut node Node) ipaddr_pub_get() !string { + if !node.done_exists('ipaddr') { + cmd := 'dig @resolver4.opendns.com myip.opendns.com +short' + res := node.exec(cmd: cmd, stdout: false)! + node.done_set('ipaddr', res.trim('\n').trim(' \n'))! + } + mut ipaddr := node.done_get('ipaddr') or { return 'Error: ipaddr is none' } + return ipaddr.trim('\n').trim(' \n') +} + +// cmd: cmd to execute . +// period in sec, e.g. if 3600, means will only execute this if not done yet within the hour . +// . +// ARGS: . +//``` +// struct NodeExecCmd{ +// cmd string +// period int //period in which we check when this was done last, if 0 then period is indefinite +// reset bool = true +// description string +// checkkey string //if used will use this one in stead of hash of cmd, to check if was executed already +// } +// ``` +pub fn (mut node Node) exec_cmd(args_ NodeExecCmd) !string { + // console.print_debug(args) + mut args := args_ + mut cmd := args.cmd + mut now_epoch := time.now().unix() + mut now_str := now_epoch.str() + if cmd.contains('\n') { + cmd = texttools.dedent(cmd) + } + + mut hhash := '' + if args.checkkey.len > 0 { + hhash = args.checkkey + } else { + hhash = md5.hexhash(cmd) + } + mut description := args.description + if description == '' { + description = cmd + if description.contains('\n') { + description = '\n${description}\n' + } + } + if !args.reset && node.done_exists('exec_${hhash}') { + if args.period == 0 { + console.print_debug(' - exec cmd:${description} on ${node.name}: was already done, period indefinite.') + return node.done_get('exec_${hhash}') or { '' } + } + nodedone := node.done_get_str('exec_${hhash}') + splitted := nodedone.split('|') + if splitted.len != 2 { + panic("Cannot return from done on exec needs to have |, now \n'${nodedone}' ") + } + exec_last_time := splitted[0].int() + lastoutput := splitted[1] + assert exec_last_time > 10000 + // console.print_debug(args) + // console.print_debug(" - check exec cmd:$cmd on $node.name: time:$exec_last_time") + if exec_last_time > now_epoch - args.period { + hours := args.period / 3600 + console.print_header('exec cmd:${description} on ${node.name}: was already done, period ${hours} h') + return lastoutput + } + } + + if args.tmpdir.len == 0 { + if 'TMPDIR' !in node.environment { + args.tmpdir = '/tmp' + } else { + args.tmpdir = node.environment['TMPDIR'] + } + } + now := ourtime.now() + r_path := '${args.tmpdir}/installer_${args.name}_${now.key()}.sh' + + node.file_write(r_path, cmd)! + cmd = "mkdir -p ${args.tmpdir} && cd ${args.tmpdir} && export TMPDIR='${args.tmpdir}' && bash ${r_path}" + + if args.remove_installer { + cmd += ' && rm -f ${r_path}' + } + $if debug { + console.print_header('exec ${r_path} on ${node.name}') + } + res := node.exec(cmd: cmd, stdout: args.stdout) or { + return error('cmd:\n${args.cmd}\n${err.msg()}') + } + + node.done_set('exec_${hhash}', '${now_str}|${res}')! + return res +} + +// check if we can execute and there is not errorcode +pub fn (mut node Node) exec_ok(cmd string) bool { + // TODO: need to put in support for multiline text files + if cmd.contains('\n') { + panic('cannot have \\n in cmd. ${cmd}, use exec function instead') + } + node.exec_silent(cmd) or { + // see if it executes ok, if cmd not found is false + return false + } + // console.print_debug(e) + return true +} + +fn (mut node Node) platform_load() ! { + console.print_header('platform load ${node.name}') + cmd := ' + if [[ "\$OSTYPE" == "darwin"* ]]; then + export OSNAME="darwin" + elif [ -e /etc/os-release ]; then + # Read the ID field from the /etc/os-release file + export OSNAME=$(grep \'^ID=\' /etc/os-release | cut -d= -f2) + if [ "\${os_id,,}" == "ubuntu" ]; then + export OSNAME="ubuntu" + fi + if [ "\${OSNAME}" == "archarm" ]; then + export OSNAME="arch" + fi + else + echo "Unable to determine the operating system." + exit 1 + fi + export HOSTNAME=\$(hostname) + export UNAME=\$(uname -m) + echo ***\${OSNAME}:\${UNAME}:\${HOSTNAME} + + ' + out := node.exec_cmd(cmd: cmd, name: 'platform_load', stdout: false)! + + out2 := out.split_into_lines().map(if it.starts_with('***') { it.trim_left('*') } else { '' }).first() + if out2.count(':') != 2 { + panic('platform loader bug') + } + splitted := out2.split(':').map(it.trim_space()) + osname := splitted[0] + cputype := splitted[1] + node.hostname = splitted[2] + + if cputype == 'x86_64' { + node.cputype = CPUType.intel + } else if cputype == 'arm64' || cputype == 'aarch64' { + node.cputype = CPUType.arm + } else { + return error("did not find cpu type, implement more types e.g. 32 bit. found: '${cputype}'") + } + + if osname == 'arch' { + node.platform = PlatformType.arch + } else if osname == 'ubuntu' || osname == 'debian' { + node.platform = PlatformType.ubuntu + } else if osname == 'darwin' { + node.platform = PlatformType.osx + } else if osname == 'alpine' { + node.platform = PlatformType.alpine + } else { + console.print_stderr(osname) + panic('only ubuntu, arch and osx supported for now') + } + console.print_debug('platform loaded') +} + +pub fn (mut node Node) package_refresh() ! { + if node.platform == PlatformType.ubuntu { + console.print_header('package refresh') + node.exec(cmd: 'apt-get update', stdout: false) or { + return error('could not update packages list\nerror:\n${err}') + } + } +} + +pub fn (mut node Node) package_install(package Package) ! { + name := package.name + if node.platform == PlatformType.osx { + node.exec(cmd: 'brew install ${name}') or { + return error('could not install package:${package.name}\nerror:\n${err}') + } + } else if node.platform == PlatformType.ubuntu { + node.exec(cmd: 'apt install -y ${name}') or { + return error('could not install package:${package.name}\nerror:\n${err}') + } + } else if node.platform == PlatformType.alpine { + node.exec(cmd: 'apk install ${name}') or { + return error('could not install package:${package.name}\nerror:\n${err}') + } + } else if node.platform == PlatformType.alpine { + node.exec(cmd: 'pacman -Su ${name}') or { + return error('could not install package:${package.name}\nerror:\n${err}') + } + } else { + panic('only ubuntu, alpine,arch and osx supported for now') + } +} diff --git a/lib/builder/node_executor.v b/lib/builder/node_executor.v new file mode 100644 index 00000000..4ba7607a --- /dev/null +++ b/lib/builder/node_executor.v @@ -0,0 +1,289 @@ +module builder + +import time + +// exec(cmd string) !string +// exec_silent(cmd string) !string +// file_write(path string, text string) ! +// file_read(path string) !string +// file_exists(path string) bool +// delete(path string) ! + +pub fn (mut node Node) exec(args ExecArgs) !string { + if mut node.executor is ExecutorLocal { + return node.executor.exec(cmd: args.cmd, stdout: args.stdout) + } else if mut node.executor is ExecutorSSH { + return node.executor.exec(cmd: args.cmd, stdout: args.stdout) + } + panic('did not find right executor') +} + +// @[params] +// pub struct ExecFileArgs { +// pub mut: +// path string +// cmd string +// } + +// pub fn (mut node Node) exec_file(args_ ExecFileArgs) !string { +// mut args:=args_ +// if args.path == '' { +// now := ourtime.now() +// args.path="/tmp/myexec_${now.key()}.sh" +// } +// if args.cmd == '' { +// return error('need to specify cmd') +// } +// args.cmd = texttools.dedent(args.cmd) +// node.file_write(args.path, args.cmd)! +// return node.exec_silent('chmod +x ${args.path} && bash ${args.path} && rm -f ${args.path}')! +// // +// } + +@[params] +pub struct ExecRetryArgs { +pub: + cmd string + retrymax int = 10 // how may times maximum to retry + period_milli int = 100 // sleep in between retry in milliseconds + timeout int = 2 // timeout for al the tries together + stdout bool = true +} + +// a cool way to execute something until it succeeds +// params: +// cmd string +// retrymax int = 10 //how may times maximum to retry +// period_milli int = 100 //sleep in between retry in milliseconds +// timeout int = 2 //timeout for al the tries together +pub fn (mut node Node) exec_retry(args ExecRetryArgs) !string { + start_time := time.now().unix_milli() + mut run_time := 0.0 + for true { + run_time = time.now().unix_milli() + if run_time > start_time + args.timeout * 1000 { + return error('timeout on exec retry for ${args}') + } + // console.print_debug(args.cmd) + r := node.exec(cmd: args.cmd, stdout: args.stdout) or { 'error' } + if r != 'error' { + return r + } + time.sleep(args.period_milli * time.millisecond) + } + return error('couldnt execute exec retry for ${args}, got at end') +} + +// silently execute a command +pub fn (mut node Node) exec_silent(cmd string) !string { + if mut node.executor is ExecutorLocal { + return node.executor.exec(cmd: cmd, stdout: false) + } else if mut node.executor is ExecutorSSH { + return node.executor.exec(cmd: cmd, stdout: false) + } + panic('did not find right executor') +} + +pub fn (mut node Node) exec_interactive(cmd_ string) ! { + if mut node.executor is ExecutorLocal { + node.executor.exec_interactive(cmd: cmd_)! + } else if mut node.executor is ExecutorSSH { + node.executor.exec_interactive(cmd: cmd_)! + } + panic('did not find right executor') +} + +pub fn (mut node Node) file_write(path string, text string) ! { + if mut node.executor is ExecutorLocal { + return node.executor.file_write(path, text) + } else if mut node.executor is ExecutorSSH { + return node.executor.file_write(path, text) + } + panic('did not find right executor') +} + +pub fn (mut node Node) file_read(path string) !string { + if mut node.executor is ExecutorLocal { + return node.executor.file_read(path) + } else if mut node.executor is ExecutorSSH { + return node.executor.file_read(path) + } + panic('did not find right executor') +} + +pub fn (mut node Node) file_exists(path string) bool { + if mut node.executor is ExecutorLocal { + return node.executor.file_exists(path) + } else if mut node.executor is ExecutorSSH { + return node.executor.file_exists(path) + } + panic('did not find right executor') +} + +// checks if given executable exists in node +pub fn (mut node Node) command_exists(cmd string) bool { + output := node.exec_silent(' + if command -v ${cmd} &> /dev/null + then + echo 0 + fi') or { + panic("can't execute command") + } + return output.contains('0') +} + +pub fn (mut node Node) delete(path string) ! { + if mut node.executor is ExecutorLocal { + return node.executor.delete(path) + } else if mut node.executor is ExecutorSSH { + return node.executor.delete(path) + } + panic('did not find right executor') +} + +@[params] +pub struct SyncArgs { +pub mut: + source string + dest string + delete bool // do we want to delete the destination + ipaddr string // e.g. root@192.168.5.5:33 (can be without root@ or :port) + ignore []string // arguments to ignore e.g. ['*.pyc','*.bak'] + ignore_default bool = true // if set will ignore a common set + stdout bool = true + fast_rsync bool = true +} + +// download files using rsync (can be ssh or local) . +// args: . +// ``` +// source string +// dest string +// delete bool //do we want to delete the destination +// ignore []string //arguments to ignore e.g. ['*.pyc','*.bak'] +// ignore_default bool = true //if set will ignore a common set +// stdout bool = true +// ``` +// . +pub fn (mut node Node) download(args_ SyncArgs) ! { + mut args := args_ + if args.source.contains('~') { + myenv := node.environ_get()! + if 'HOME' !in myenv { + return error('Cannot find home in env for ${node}') + } + mut myhome := myenv['HOME'] + args.source.replace('~', myhome) + } + if mut node.executor is ExecutorLocal { + return node.executor.download(args) + } else if mut node.executor is ExecutorSSH { + return node.executor.download(args) + } + panic('did not find right executor') +} + +// upload files using rsync (can be ssh or local) +// args: . +// ``` +// source string +// dest string +// delete bool //do we want to delete the destination +// ignore []string //arguments to ignore e.g. ['*.pyc','*.bak'] +// ignore_default bool = true //if set will ignore a common set +// stdout bool = true +// ``` +// . +pub fn (mut node Node) upload(args_ SyncArgs) ! { + mut args := args_ + if args.dest.contains('~') { + myenv := node.environ_get()! + if 'HOME' !in myenv { + return error('Cannot find home in env for ${node}') + } + mut myhome := myenv['HOME'] + args.dest.replace('~', myhome) + } + if mut node.executor is ExecutorLocal { + return node.executor.upload(args) + } else if mut node.executor is ExecutorSSH { + return node.executor.upload(args) + } + panic('did not find right executor') +} + +@[params] +pub struct EnvGetParams { +pub mut: + reload bool +} + +pub fn (mut node Node) environ_get(args EnvGetParams) !map[string]string { + if args.reload { + if mut node.executor is ExecutorLocal { + return node.executor.environ_get() + } else if mut node.executor is ExecutorSSH { + return node.executor.environ_get() + } + panic('did not find right executor') + } + return node.environment +} + +pub fn (mut node Node) info() map[string]string { + if mut node.executor is ExecutorLocal { + return node.executor.info() + } else if mut node.executor is ExecutorSSH { + return node.executor.info() + } + panic('did not find right executor') +} + +pub fn (mut node Node) shell(cmd string) ! { + if mut node.executor is ExecutorLocal { + return node.executor.shell(cmd) + } else if mut node.executor is ExecutorSSH { + return node.executor.shell(cmd) + } + panic('did not find right executor') +} + +// list(path string) ![]string +// dir_exists(path string) bool +// debug_off() +// debug_on() +pub fn (mut node Node) list(path string) ![]string { + if mut node.executor is ExecutorLocal { + return node.executor.list(path) + } else if mut node.executor is ExecutorSSH { + return node.executor.list(path) + } + panic('did not find right executor') +} + +pub fn (mut node Node) dir_exists(path string) bool { + if mut node.executor is ExecutorLocal { + return node.executor.dir_exists(path) + } else if mut node.executor is ExecutorSSH { + return node.executor.dir_exists(path) + } + panic('did not find right executor') +} + +pub fn (mut node Node) debug_off() { + if mut node.executor is ExecutorLocal { + node.executor.debug_off() + } else if mut node.executor is ExecutorSSH { + node.executor.debug_off() + } + panic('did not find right executor') +} + +pub fn (mut node Node) debug_on() { + if mut node.executor is ExecutorLocal { + node.executor.debug_on() + } else if mut node.executor is ExecutorSSH { + node.executor.debug_on() + } + panic('did not find right executor') +} diff --git a/lib/builder/node_factory.v b/lib/builder/node_factory.v new file mode 100644 index 00000000..7b2e39b9 --- /dev/null +++ b/lib/builder/node_factory.v @@ -0,0 +1,75 @@ +module builder + +import freeflowuniverse.herolib.data.ipaddress + +// get node connection to local machine +pub fn (mut bldr BuilderFactory) node_local() !&Node { + return bldr.node_new(name: 'localhost') +} + +// format ipaddr: localhost:7777 . +// format ipaddr: 192.168.6.6:7777 . +// format ipaddr: 192.168.6.6 . +// format ipaddr: any ipv6 addr . +// format ipaddr: if only name used then is localhost . +@[params] +pub struct NodeArguments { +pub mut: + ipaddr string + name string + user string = 'root' + debug bool + reload bool +} + +// the factory which returns an node, based on the arguments will chose ssh executor or the local one +// . +//``` +// - format ipaddr: localhost:7777 +// - format ipaddr: myuser@192.168.6.6:7777 +// - format ipaddr: 192.168.6.6 +// - format ipaddr: any ipv6 addr +// - if only name used then is localhost with localhost executor +//``` +// its possible to put a user as user@ .. in front of an ip addr . +// . +//``` +// pub struct NodeArguments { +// ipaddr string +// name string //if not filled in will come from ipaddr +// user string = "root" +// debug bool +// reset bool +// } +//``` +pub fn (mut bldr BuilderFactory) node_new(args_ NodeArguments) !&Node { + mut args := args_ + if args.ipaddr.contains('@') { + args.user, args.ipaddr = args.ipaddr.split_once('@') or { panic('bug') } + } + + eargs := ExecutorNewArguments{ + ipaddr: args.ipaddr + user: args.user + debug: args.debug + } + mut executor := executor_new(eargs)! + mut node := Node{ + name: args.name + executor: executor + factory: &bldr + } + + if node.name == '' { + mut iadd := ipaddress.new(args.ipaddr)! + node.name = iadd.toname()! + } + + wasincache := node.load()! + + if wasincache && args.reload { + node.readfromsystem()! + } + + return &node +} diff --git a/lib/builder/nodedb_test.v b/lib/builder/nodedb_test.v new file mode 100644 index 00000000..f62e31ca --- /dev/null +++ b/lib/builder/nodedb_test.v @@ -0,0 +1,9 @@ +module builder + +fn test_nodedb() { + // TODO URGENT create tests for nodedb +} + +fn test_nodedone() { + // TODO URGENT create tests for nodedone +} diff --git a/lib/builder/portforward_lib.v b/lib/builder/portforward_lib.v new file mode 100644 index 00000000..1fb78030 --- /dev/null +++ b/lib/builder/portforward_lib.v @@ -0,0 +1,31 @@ +module builder + +import freeflowuniverse.herolib.osal.screen +import freeflowuniverse.herolib.data.ipaddress +import freeflowuniverse.herolib.ui.console + +@[params] +pub struct ForwardArgsToLocal { +pub mut: + name string @[required] + address string @[required] + remote_port int @[required] + local_port int + user string = 'root' +} + +// forward a remote port on ssh host to a local port +pub fn portforward_to_local(args_ ForwardArgsToLocal) ! { + mut args := args_ + if args.local_port == 0 { + args.local_port = args.remote_port + } + mut addr := ipaddress.new(args.address)! + mut cmd := 'ssh -L ${args.local_port}:localhost:${args.remote_port} ${args.user}@${args.address}' + if addr.cat == .ipv6 { + cmd = 'ssh -L ${args.local_port}:localhost:${args.remote_port} ${args.user}@${args.address.trim('[]')}' + } + console.print_debug(cmd) + mut scr := screen.new(reset: false)! + _ = scr.add(name: args.name, cmd: cmd)! +} diff --git a/lib/builder/readme.md b/lib/builder/readme.md new file mode 100644 index 00000000..e7ac6b7d --- /dev/null +++ b/lib/builder/readme.md @@ -0,0 +1,165 @@ +# Builder Module + +The Builder module is a powerful system automation and remote execution framework that provides a unified interface for executing commands and managing files across both local and remote systems. + +## Overview + +The Builder module consists of several key components: +- **BuilderFactory**: Creates and manages builder instances +- **Node**: Represents a target system (local or remote) with its properties and state +- **Executor**: Interface for command execution and file operations (SSH or Local) +- **NodeDB**: Key-value store at the node level for persistent state + +## Getting Started + +### Basic Initialization + +```v +import freeflowuniverse.herolib.builder + +// Create a new builder instance +mut b := builder.new()! + +// Create a node for remote execution +mut n := b.node_new(ipaddr: "root@195.192.213.2:2222")! + +// Or create a local node +mut local_node := builder.node_local()! +``` + +### Node Configuration + +Nodes can be configured with various properties: +```v +// Full node configuration +mut n := b.node_new( + name: "myserver", // Optional name for the node + ipaddr: "root@server.example.com:22", // SSH connection string + platform: .ubuntu, // Target platform type + debug: true // Enable debug output +)! +``` + +## Node Properties + +Each node maintains information about: +- Platform type (OSX, Ubuntu, Alpine, Arch) +- CPU architecture (Intel, ARM) +- Environment variables +- System state +- Execution history + +The node automatically detects and caches system information for better performance. + +## Executor Interface + +The executor provides a unified interface for both local and remote operations: + +### Command Execution +```v +// Execute command and get output +result := n.exec("ls -la")! + +// Execute silently (no output) +n.exec_silent("mkdir -p /tmp/test")! + +// Interactive shell +n.shell("bash")! +``` + +### File Operations +```v +// Write file +n.file_write("/path/to/file", "content")! + +// Read file +content := n.file_read("/path/to/file")! + +// Check existence +exists := n.file_exists("/path/to/file") + +// Delete file/directory +n.delete("/path/to/delete")! + +// List directory contents +files := n.list("/path/to/dir")! + +// File transfers +n.download("http://example.com/file", "/local/path")! +n.upload("/local/file", "/remote/path")! +``` + +### Environment Management +```v +// Get all environment variables +env := n.environ_get()! + +// Get node information +info := n.info() +``` + +## Node Database (NodeDB) + +The NodeDB provides persistent key-value storage at the node level: + +```v +// Store a value +n.done["key"] = "value" +n.save()! + +// Load stored values +n.load()! +value := n.done["key"] +``` + +This is useful for: +- Caching system information +- Storing configuration state +- Tracking execution history +- Maintaining persistent data between sessions + +## Best Practices + +1. **Error Handling**: Always use the `!` operator for methods that can fail and handle errors appropriately. + +2. **Resource Management**: Close connections and clean up resources when done: +```v +defer { + n.close() +} +``` + +3. **Debug Mode**: Enable debug mode when troubleshooting: +```v +n.debug_on() // Enable debug output +n.debug_off() // Disable debug output +``` + +4. **Platform Awareness**: Check platform compatibility before executing commands: +```v +if n.platform == .ubuntu { + // Ubuntu-specific commands +} else if n.platform == .osx { + // macOS-specific commands +} +``` + +## Examples + +See complete examples in: +- Simple usage: `examples/builder/simple.vsh` +- Remote execution: `examples/builder/remote_executor/` +- Platform-specific examples: + - IPv4: `examples/builder/simple_ip4.vsh` + - IPv6: `examples/builder/simple_ip6.vsh` + +## Implementation Details + +The Builder module uses: +- Redis for caching node information +- SSH for secure remote execution +- MD5 hashing for unique node identification +- JSON for data serialization +- Environment detection for platform-specific behavior + +For more detailed implementation information, refer to the source code in the `lib/builder/` directory. diff --git a/lib/builder/this_remote.v b/lib/builder/this_remote.v new file mode 100644 index 00000000..a1b70226 --- /dev/null +++ b/lib/builder/this_remote.v @@ -0,0 +1,51 @@ +module builder + +import os +import freeflowuniverse.herolib.core.texttools +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.ui.console + +@[params] +pub struct ThisRemoteArgs { +pub mut: + name string = 'remote' + nodes string + script string + sync_from_local bool +} + +// to use do something like: export NODES="195.192.213.3" . +pub fn this_remote_exec(args_ ThisRemoteArgs) !bool { + mut args := args_ + if args.script.trim_space().starts_with('/tmp/remote_') { + return false // means we need to execute + } + addr := texttools.to_array(args.nodes) + // console.print_debug(addr) + mut counter := 0 + for a in addr { + counter += 1 + name := '${args.name}_${counter}' + mut b := new()! + if args.sync_from_local { + mut n := b.node_new(ipaddr: a, name: name)! + // console.print_debug(n.ipaddr_pub_get()!) + n.vscript(path: args.script, sync_from_local: args.sync_from_local)! + } else { + // is a shortcut goes faster if no update + if !os.exists(args.script) { + return error("can't find script ${args.script}") + } + cmd := 'scp ${args.script} ${a}:/tmp/remote_${name}.vsh' + console.print_debug(cmd) + r := os.execute(cmd) + if r.exit_code > 0 { + return error('could not scp: ${cmd}') + } + // cmd:="scp ${args.script} ${a}:/tmp/remote_${name}.vsh" + cmd2 := "ssh ${a} 'v -w -enable-globals /tmp/remote_${name}.vsh'" + osal.exec(cmd: cmd2, stdout: true)! + } + } + return true +} diff --git a/lib/core/playcmds/_archive/play_juggler.v b/lib/core/playcmds/_archive/play_juggler.v new file mode 100644 index 00000000..6d5fbd2c --- /dev/null +++ b/lib/core/playcmds/_archive/play_juggler.v @@ -0,0 +1,47 @@ +module playcmds + +import freeflowuniverse.herolib.data.doctree +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.develop.juggler +import os + +pub fn play_juggler(mut plbook playbook.PlayBook) ! { + mut coderoot := '' + // mut install := false + mut reset := false + mut pull := false + + mut config_actions := plbook.find(filter: 'juggler.configure')! + + mut j := juggler.Juggler{} + + if config_actions.len > 1 { + return error('can only have 1 config action for juggler') + } else if config_actions.len == 1 { + mut p := config_actions[0].params + path := p.get_default('path', '/etc/juggler')! + url := p.get_default('url', '')! + username := p.get_default('username', '')! + password := p.get_default('password', '')! + port := p.get_int_default('port', 8000)! + + j = juggler.configure( + url: 'https://git.ourworld.tf/projectmycelium/itenv' + username: username + password: password + reset: true + )! + config_actions[0].done = true + } + + for mut action in plbook.find(filter: 'juggler.start')! { + j.start()! + action.done = true + } + + for mut action in plbook.find(filter: 'juggler.restart')! { + j.restart()! + action.done = true + } +} diff --git a/lib/core/playcmds/bizmodel.v b/lib/core/playcmds/bizmodel.v new file mode 100644 index 00000000..b2096b27 --- /dev/null +++ b/lib/core/playcmds/bizmodel.v @@ -0,0 +1,19 @@ +module playcmds + +// import freeflowuniverse.herolib.core.playbook + +// fn git(mut actions playbook.Actions, action playbook.Action) ! { +// if action.name == 'init' { +// // means we support initialization afterwards +// c.bizmodel_init(mut actions, action)! +// } + +// // if action.name == 'get' { +// // mut gs := gittools.get()! +// // url := action.params.get('url')! +// // branch := action.params.get_default('branch', '')! +// // reset := action.params.get_default_false('reset')! +// // pull := action.params.get_default_false('pull')! +// // mut gr := gs.repo_get_from_url(url: url, branch: branch, pull: pull, reset: reset)! +// // } +// } diff --git a/lib/core/playcmds/currency.v b/lib/core/playcmds/currency.v new file mode 100644 index 00000000..d86739b8 --- /dev/null +++ b/lib/core/playcmds/currency.v @@ -0,0 +1,21 @@ +module playcmds + +// fn currency_actions(actions_ []playbook.Action) ! { +// mut actions2 := actions.filtersort(actions: actions_, actor: 'currency', book: '*')! +// if actions2.len == 0 { +// return +// } + +// mut cs := currency.new()! + +// for action in actions2 { +// // TODO: set the currencies +// if action.name == 'default_set' { +// cur := action.params.get('cur')! +// usdval := action.params.get_int('usdval')! +// cs.default_set(cur, usdval)! +// } +// } + +// // TODO: add the currency metainfo, do a test +// } diff --git a/lib/core/playcmds/downloader.v b/lib/core/playcmds/downloader.v new file mode 100644 index 00000000..0fe55d19 --- /dev/null +++ b/lib/core/playcmds/downloader.v @@ -0,0 +1,59 @@ +module playcmds + +// import freeflowuniverse.herolib.core.playbook +// import freeflowuniverse.herolib.sysadmin.downloader + +// can start with sal, dal, ... the 2nd name is typicall the actor (or topic) +// do this function public and then it breaches out to detail functionality + +// pub fn sal_downloader(action playbook.Action) ! { +// match action.actor { +// 'downloader' { +// match action.name { +// 'get' { +// downloader_get(action: action)! +// } +// else { +// return error('actions not supported yet') +// } +// } +// } +// else { +// return error('actor not supported yet') +// } +// } +// } + +// fn downloader_get(args ActionExecArgs) ! { +// action := args.action +// // session:=args.action or {panic("no context")} //if we need it here +// mut name := action.params.get_default('name', '')! +// mut downloadpath := action.params.get_default('downloadpath', '')! +// mut url := action.params.get_default('url', '')! +// mut reset := action.params.get_default_false('reset') +// mut gitpull := action.params.get_default_false('gitpull') + +// mut minsize_kb := action.params.get_u32_default('minsize_kb', 0)! +// mut maxsize_kb := action.params.get_u32_default('maxsize_kb', 0)! + +// mut destlink := action.params.get_default_false('destlink') + +// mut dest := action.params.get_default('dest', '')! +// mut hash := action.params.get_default('hash', '')! +// mut metapath := action.params.get_default('metapath', '')! + +// mut meta := downloader.download( +// name: name +// downloadpath: downloadpath +// url: url +// reset: reset +// gitpull: gitpull +// minsize_kb: minsize_kb +// maxsize_kb: maxsize_kb +// destlink: destlink +// dest: dest +// hash: hash +// metapath: metapath +// // session:session // TODO IMPLEMENT (also optional) +// )! +// } diff --git a/lib/core/playcmds/factory.v b/lib/core/playcmds/factory.v new file mode 100644 index 00000000..d075a35a --- /dev/null +++ b/lib/core/playcmds/factory.v @@ -0,0 +1,44 @@ +module playcmds + +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.virt.hetzner +//import freeflowuniverse.herolib.clients.b2 +import freeflowuniverse.herolib.biz.bizmodel +import freeflowuniverse.herolib.hero.publishing +import freeflowuniverse.herolib.threefold.grid4.gridsimulator +//import freeflowuniverse.herolib.installers.sysadmintools.daguserver +import freeflowuniverse.herolib.threefold.grid4.farmingsimulator +import freeflowuniverse.herolib.web.components.slides +// import freeflowuniverse.herolib.installers.base as base_install +// import freeflowuniverse.herolib.installers.infra.coredns + +pub fn run(mut plbook playbook.PlayBook, dagu bool) ! { + if dagu { + hscript := plbook.str() + scheduler(hscript)! + } + + play_core(mut plbook)! + play_ssh(mut plbook)! + play_git(mut plbook)! + //play_zola(mut plbook)! + //play_caddy(mut plbook)! + //play_juggler(mut plbook)! + //play_luadns(mut plbook)! + hetzner.heroplay(mut plbook)! + //b2.heroplay(mut plbook)! + + farmingsimulator.play(mut plbook)! + gridsimulator.play(mut plbook)! + bizmodel.play(mut plbook)! + slides.play(mut plbook)! + // base_install(play(mut plbook)! + // coredns.play(mut plbook)! + + publishing.play(mut plbook)! + + //plbook.empty_check()! + + console.print_header('Actions concluded succesfully.') +} diff --git a/lib/core/playcmds/play_caddy.v b/lib/core/playcmds/play_caddy.v new file mode 100644 index 00000000..c59807ea --- /dev/null +++ b/lib/core/playcmds/play_caddy.v @@ -0,0 +1,143 @@ +module playcmds + +// import freeflowuniverse.herolib.installers.web.caddy as caddy_installer +// import freeflowuniverse.herolib.servers.caddy { CaddyFile } +// import freeflowuniverse.herolib.core.playbook +// import os +// // import net.urllib + +// pub fn play_caddy(mut plbook playbook.PlayBook) ! { +// play_caddy_basic(mut plbook)! +// play_caddy_configure(mut plbook)! +// } + +// pub fn play_caddy_configure(mut plbook playbook.PlayBook) ! { +// mut caddy_actions := plbook.find(filter: 'caddy_configure')! +// if caddy_actions.len == 0 { +// return +// } +// } + +// pub fn play_caddy_basic(mut plbook playbook.PlayBook) ! { +// caddy_actions := plbook.find(filter: 'caddy.')! +// if caddy_actions.len == 0 { +// return +// } + +// mut install_actions := plbook.find(filter: 'caddy.install')! + +// if install_actions.len > 0 { +// for install_action in install_actions { +// mut p := install_action.params +// xcaddy := p.get_default_false('xcaddy') +// file_path := p.get_default('file_path', '/etc/caddy')! +// file_url := p.get_default('file_url', '')! +// reset := p.get_default_false('reset') +// start := p.get_default_false('start') +// restart := p.get_default_false('restart') +// stop := p.get_default_false('stop') +// homedir := p.get_default('file_url', '')! +// plugins := p.get_list_default('plugins', []string{})! + +// caddy_installer.install( +// xcaddy: xcaddy +// file_path: file_path +// file_url: file_url +// reset: reset +// start: start +// restart: restart +// stop: stop +// homedir: homedir +// plugins: plugins +// )! +// } +// } + +// mut config_actions := plbook.find(filter: 'caddy.configure')! +// if config_actions.len > 0 { +// mut coderoot := '' +// mut reset := false +// mut pull := false + +// mut public_ip := '' + +// mut c := caddy.get('')! +// // that to me seems to be wrong, not generic enough +// if config_actions.len > 1 { +// return error('can only have 1 config action for books') +// } else if config_actions.len == 1 { +// mut p := config_actions[0].params +// path := p.get_default('path', '/etc/caddy')! +// url := p.get_default('url', '')! +// public_ip = p.get_default('public_ip', '')! +// c = caddy.configure('', homedir: path)! +// config_actions[0].done = true +// } + +// mut caddyfile := CaddyFile{} +// for mut action in plbook.find(filter: 'caddy.add_reverse_proxy')! { +// mut p := action.params +// mut from := p.get_default('from', '')! +// mut to := p.get_default('to', '')! + +// if from == '' || to == '' { +// return error('from & to cannot be empty') +// } + +// caddyfile.add_reverse_proxy( +// from: from +// to: to +// )! +// action.done = true +// } + +// for mut action in plbook.find(filter: 'caddy.add_file_server')! { +// mut p := action.params +// mut domain := p.get_default('domain', '')! +// mut root := p.get_default('root', '')! + +// if root.starts_with('~') { +// root = '${os.home_dir()}${root.trim_string_left('~')}' +// } + +// if domain == '' || root == '' { +// return error('domain & root cannot be empty') +// } + +// caddyfile.add_file_server( +// domain: domain +// root: root +// )! +// action.done = true +// } + +// for mut action in plbook.find(filter: 'caddy.add_basic_auth')! { +// mut p := action.params +// mut domain := p.get_default('domain', '')! +// mut username := p.get_default('username', '')! +// mut password := p.get_default('password', '')! + +// if domain == '' || username == '' || password == '' { +// return error('domain & root cannot be empty') +// } + +// caddyfile.add_basic_auth( +// domain: domain +// username: username +// password: password +// )! +// action.done = true +// } + +// for mut action in plbook.find(filter: 'caddy.generate')! { +// c.set_caddyfile(caddyfile)! +// action.done = true +// } + +// for mut action in plbook.find(filter: 'caddy.start')! { +// c.start()! +// action.done = true +// } +// c.reload()! +// } +// } diff --git a/lib/core/playcmds/play_core.v b/lib/core/playcmds/play_core.v new file mode 100644 index 00000000..cad378a8 --- /dev/null +++ b/lib/core/playcmds/play_core.v @@ -0,0 +1,92 @@ +module playcmds + +import freeflowuniverse.herolib.develop.gittools +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.ui.console + +// !!context.configure +// name:'test' +// coderoot:... +// interactive:true + +pub fn play_core(mut plbook playbook.PlayBook) ! { + for mut action in plbook.find(filter: 'context.configure')! { + mut p := action.params + mut session := plbook.session + + if p.exists('interactive') { + session.interactive = p.get_default_false('interactive') + } + + if p.exists('coderoot') { + panic('implement') + mut coderoot := p.get_path_create('coderoot')! + + mut gs := gittools.get()! + } + action.done = true + } + + for mut action in plbook.find(filter: 'session.')! { + mut p := action.params + mut session := plbook.session + + //!!session.env_set key:'JWT_SHARED_KEY' val:'...' + + action.done = true + } + + for action_ in plbook.find(filter: 'play.*')! { + if action_.name == 'run' { + console.print_debug('play run:${action_}') + mut action := *action_ + mut playrunpath := action.params.get_default('path', '')! + if playrunpath.len == 0 { + action.name = 'pull' + action2 := play_git_action(action)! + playrunpath = action2.params.get_default('path', '')! + } + if playrunpath.len == 0 { + return error("can't run a heroscript didn't find url or path.") + } + console.print_debug('play run path:${playrunpath}') + plbook.add(path: playrunpath)! + } + if action_.name == 'echo' { + content := action_.params.get_default('content', "didn't find content")! + console.print_header(content) + } + } + + // for mut action in plbook.find(filter: 'core.coderoot_set')! { + // mut p := action.params + // if p.exists('coderoot') { + // coderoot := p.get_path_create('coderoot')! + // mut gs := session.context.gitstructure()! + // if gs.rootpath.path != coderoot { + // mut db := session.context.contextdb.db_get(dbname: 'context')! + // db.set('coderoot', coderoot)! + // session.context.gitstructure_reload()! + // } + // } else { + // return error('coderoot needs to be specified') + // } + // action.done = true + // } + + // for mut action in plbook.find(filter: 'core.params_context_set')! { + // mut p := action.params + // for param in p.params { + // session.context.params.set(param.key, param.value) + // } + // action.done = true + // } + + // for mut action in plbook.find(filter: 'core.params_session_set')! { + // mut p := action.params + // for param in p.params { + // session.params.set(param.key, param.value) + // } + // action.done = true + // } +} diff --git a/lib/core/playcmds/play_dagu.v b/lib/core/playcmds/play_dagu.v new file mode 100644 index 00000000..5986d323 --- /dev/null +++ b/lib/core/playcmds/play_dagu.v @@ -0,0 +1,95 @@ +module playcmds + +import freeflowuniverse.herolib.clients.daguclient +import freeflowuniverse.herolib.installers.sysadmintools.daguserver +// import freeflowuniverse.herolib.installers.sysadmintools.daguserver +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.ui.console +import os + +pub fn play_dagu(mut plbook playbook.PlayBook) ! { + // dagu_actions := plbook.find(filter: 'dagu.')! + // if dagu_actions.len == 0 { + // return + // } + + // play_dagu_basic(mut plbook)! + // play_dagu_configure(mut plbook)! +} + +// play_dagu plays the dagu play commands +pub fn play_dagu_basic(mut plbook playbook.PlayBook) ! { + // mut install_actions := plbook.find(filter: 'daguserver.configure')! + + // if install_actions.len > 0 { + // for install_action in install_actions { + // mut p := install_action.params + // panic("daguinstall play") + // } + // } + + // dagu_actions := plbook.find(filter: 'daguserver.install')! + // if dagu_actions.len > 0 { + // panic("daguinstall play") + // return + // } + + // mut config_actions := plbook.find(filter: 'dagu.configure')! + // mut d := if config_actions.len > 1 { + // return error('can only have 1 config action for dagu') + // } else if config_actions.len == 1 { + // mut p := config_actions[0].params + // instance := p.get_default('instance', 'default')! + // port := p.get_int_default('port', 8888)! + // username := p.get_default('username', '')! + // password := p.get_default('password', '')! + // config_actions[0].done = true + // mut server := daguserver.configure(instance, + // port: port + // username: username + // password: password + // )! + // server.start()! + // console.print_debug('Dagu server is running at http://localhost:${port}') + // console.print_debug('Username: ${username} password: ${password}') + + // // configure dagu client with server url and api secret + // server_cfg := server.config()! + // daguclient.get(instance, + // url: 'http://localhost:${port}' + // apisecret: server_cfg.secret + // )! + // } else { + // mut server := daguserver.get('')! + // server.start()! + // daguclient.get('')! + // } + + // mut dags := map[string]DAG{} + + // for mut action in plbook.find(filter: 'dagu.new_dag')! { + // mut p := action.params + // name := p.get_default('name', '')! + // dags[name] = DAG{} + // action.done = true + // } + + // for mut action in plbook.find(filter: 'dagu.add_step')! { + // mut p := action.params + // dag := p.get_default('dag', 'default')! + // name := p.get_default('name', 'default')! + // command := p.get_default('command', '')! + // dags[dag].step_add( + // nr: dags.len + // name: name + // command: command + // )! + // } + + // for mut action in plbook.find(filter: 'dagu.run')! { + // mut p := action.params + // dag := p.get_default('dag', 'default')! + // // d.new_dag(dags[dag])! + // panic('to implement') + // } +} diff --git a/lib/core/playcmds/play_dagu_test.v b/lib/core/playcmds/play_dagu_test.v new file mode 100644 index 00000000..61512d5b --- /dev/null +++ b/lib/core/playcmds/play_dagu_test.v @@ -0,0 +1,31 @@ +module playcmds + +import freeflowuniverse.herolib.core.playbook + +const dagu_script = " +!!dagu.configure + instance: 'test' + username: 'admin' + password: 'testpassword' + +!!dagu.new_dag + name: 'test_dag' + +!!dagu.add_step + dag: 'test_dag' + name: 'hello_world' + command: 'echo hello world' + +!!dagu.add_step + dag: 'test_dag' + name: 'last_step' + command: 'echo last step' + + +" + +fn test_play_dagu() ! { + mut plbook := playbook.new(text: playcmds.dagu_script)! + play_dagu(mut plbook)! + panic('s') +} diff --git a/lib/core/playcmds/play_doctree.v b/lib/core/playcmds/play_doctree.v new file mode 100644 index 00000000..ad89a101 --- /dev/null +++ b/lib/core/playcmds/play_doctree.v @@ -0,0 +1,70 @@ +module playcmds + +import freeflowuniverse.herolib.data.doctree +import freeflowuniverse.herolib.core.playbook +import os + +pub fn play_doctree(mut plbook playbook.PlayBook) ! { + + // check if any actions for doctree, if not then nothing to do here + // dtactions := plbook.find(filter: 'doctree.')! + // if dtactions.len == 0 { + // console.print_debug("can't find doctree.add statements, nothing to do") + // return + // } + + mut trees := map[string]&doctree.Tree{} + for mut action in plbook.find(filter: 'doctree:new')! { + mut p := action.params + name := p.get('name')! + fail_on_error := p.get_default_false('fail_on_error') + println('fail on error: ${fail_on_error}') + if name in trees { + return error('tree with name ${name} already exists') + } + + tree := doctree.new(name: name, fail_on_error: fail_on_error)! + trees[name] = tree + } + + for mut action in plbook.find(filter: 'doctree:add')! { + mut p := action.params + url := p.get_default('url', '')! + path := p.get_default('path', '')! + name := p.get('name')! + + mut tree := trees[name] or { return error('tree ${name} not found') } + + // tree.scan( + // path: path + // git_url: url + // git_reset: reset + // git_root: coderoot + // git_pull: pull + // )! + // action.done = true + } + + for mut action in plbook.find(filter: 'doctree:export')! { + mut p := action.params + build_path := p.get('path')! + toreplace := p.get_default('replace', '')! + reset2 := p.get_default_false('reset') + name := p.get('name')! + mut tree := trees[name] or { return error('tree: ${name} not found') } + + tree.export( + destination: build_path + reset: reset2 + toreplace: toreplace + )! + action.done = true + } + + for mut action in plbook.find(filter: 'doctree:export')! { + panic('implement') + mut p := action.params + name := p.get('name')! + action.done = true + } +} diff --git a/lib/core/playcmds/play_git.v b/lib/core/playcmds/play_git.v new file mode 100644 index 00000000..a1bba04c --- /dev/null +++ b/lib/core/playcmds/play_git.v @@ -0,0 +1,54 @@ +module playcmds + +import freeflowuniverse.herolib.develop.gittools +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.ui.console + +pub fn play_git(mut plbook playbook.PlayBook) ! { + for action in plbook.find(filter: 'gittools.*')! { + play_git_action(action)! + } +} + +pub fn play_git_action(action playbook.Action) !playbook.Action { + // console.print_debug("play git action: ${action}") + mut p := action.params + mut repo := p.get_default('repo', '')! + mut account := p.get_default('account', '')! + mut provider := p.get_default('provider', '')! + // mut filter := p.get_default('filter', '')! + mut url := p.get_default('url', '')! + + mut cmd := action.name + + mut coderoot := '' + if p.exists('coderoot') { + coderoot = p.get_path_create('coderoot')! + } + + if (repo == '' || account == '' || provider == '') && url == '' { + return error('need to specify repo, account and provider if url is not specified') + } + + mut gs := gittools.get(coderoot: coderoot) or { + return error("Could not load gittools on '${coderoot}'\n${err}") + } + + gitpath := gs.do( + cmd: cmd + filter: action.params.get_default('filter', '')! + repo: repo + account: account + provider: provider + script: action.params.get_default_false('script') + reset: action.params.get_default_false('reset') + pull: action.params.get_default_false('pull') + msg: action.params.get_default('message', '')! + url: url + )! + console.print_debug('play git action: ${cmd} ${account}:${repo} path:${gitpath}') + mut action2 := action + action2.params.set('path', gitpath) + action2.done = true + return action2 +} diff --git a/lib/core/playcmds/play_luadns.v b/lib/core/playcmds/play_luadns.v new file mode 100644 index 00000000..350dcd17 --- /dev/null +++ b/lib/core/playcmds/play_luadns.v @@ -0,0 +1,35 @@ +module playcmds + +import freeflowuniverse.herolib.develop.luadns +import freeflowuniverse.herolib.core.playbook +import os + +pub fn play_luadns(mut plbook playbook.PlayBook) ! { + mut buildroot := '${os.home_dir()}/hero/var/mdbuild' + mut publishroot := '${os.home_dir()}/hero/www/info' + mut coderoot := '' + // mut install := false + mut reset := false + mut pull := false + + for mut action in plbook.find(filter: 'luadns.set_domain')! { + mut p := action.params + url := p.get_default('url', '')! + + if url == '' { + return error('luadns url cant be empty') + } + + mut dns := luadns.load(url)! + + domain := p.get_default('domain', '')! + ip := p.get_default('ip', '')! + + if domain == '' || ip == '' { + return error('luadns set domain: domain or ip cant be empty') + } + + dns.set_domain(domain, ip)! + action.done = true + } +} diff --git a/lib/core/playcmds/play_mdbook.v b/lib/core/playcmds/play_mdbook.v new file mode 100644 index 00000000..9262de89 --- /dev/null +++ b/lib/core/playcmds/play_mdbook.v @@ -0,0 +1,136 @@ +module playcmds + +import freeflowuniverse.herolib.web.mdbook +import freeflowuniverse.herolib.data.doctree +import freeflowuniverse.herolib.core.playbook +import os + +pub fn play_mdbook(mut plbook playbook.PlayBook) ! { + mut buildroot := '${os.home_dir()}/hero/var/mdbuild' + mut publishroot := '${os.home_dir()}/hero/www/info' + mut coderoot := '' + // mut install := false + mut reset := false + mut pull := false + + // check if any actions for doctree, if not then nothing to do here + // dtactions := plbook.find(filter: 'doctree.')! + // if dtactions.len == 0 { + // console.print_debug("can't find doctree.add statements, nothing to do") + // return + // } + + mut config_actions := plbook.find(filter: 'books:configure')! + + if config_actions.len > 1 { + return error('can only have 1 config action for books') + } else if config_actions.len == 1 { + mut p := config_actions[0].params + if p.exists('buildroot') { + buildroot = p.get('buildroot')! + } + if p.exists('coderoot') { + coderoot = p.get('coderoot')! + } + if p.exists('publishroot') { + publishroot = p.get('publishroot')! + } + if p.exists('reset') { + reset = p.get_default_false('reset') + } + config_actions[0].done = true + } + + mut trees := map[string]&doctree.Tree{} + for mut action in plbook.find(filter: 'doctree:new')! { + mut p := action.params + name := p.get('name')! + fail_on_error := p.get_default_false('fail_on_error') + if name in trees { + return error('tree with name ${name} already exists') + } + + tree := doctree.new(name: name, fail_on_error: fail_on_error)! + trees[name] = tree + } + + for mut action in plbook.find(filter: 'doctree:add')! { + mut p := action.params + url := p.get_default('url', '')! + path := p.get_default('path', '')! + name := p.get_default('name', '')! + + if trees.len == 0 { + return error('no tree found') + } + + mut tree := if name != '' { + trees[name] or { return error('tree ${name} not found') } + } else { + trees.values()[0] + } + + tree.scan( + path: path + git_url: url + git_reset: reset + git_root: coderoot + git_pull: pull + )! + action.done = true + } + + for mut action in plbook.find(filter: 'doctree:export')! { + mut p := action.params + build_path := p.get('path')! + toreplace := p.get_default('replace', '')! + reset2 := p.get_default_false('reset') + name := p.get('name')! + mut tree := trees[name] or { return error('tree: ${name} not found') } + + tree.export( + destination: build_path + reset: reset2 + toreplace: toreplace + )! + action.done = true + } + + for mut action in plbook.find(filter: 'mdbook:export')! { + mut p := action.params + name := p.get('name')! + summary_url := p.get_default('summary_url', '')! + summary_path := p.get_default('summary_path', '')! + title := p.get_default('title', name)! + publish_path := p.get_default('publish_path', '${publishroot}/${name}')! + build_path := p.get_default('build_path', '${buildroot}/${name}')! + printbook := p.get_default_false('printbook') + foldlevel := p.get_int_default('foldlevel', 0)! + production := p.get_default_false('production') + reset3 := p.get_default_true('reset') + collections := p.get_list_default('collections', [])! + + if summary_url == '' && summary_path == '' { + return error('Both summary url and path cannot be empty at the same time') + } + + mut mdbooks := mdbook.get()! + + mut cfg := mdbooks.config()! + cfg.path_build = buildroot + cfg.path_publish = publishroot + + mdbooks.generate( + name: name + title: title + summary_path: summary_path + publish_path: publish_path + build_path: build_path + printbook: printbook + foldlevel: foldlevel + production: production + collections: collections + )! + action.done = true + } +} diff --git a/lib/core/playcmds/play_mdbook_test.v b/lib/core/playcmds/play_mdbook_test.v new file mode 100644 index 00000000..51c01dad --- /dev/null +++ b/lib/core/playcmds/play_mdbook_test.v @@ -0,0 +1,85 @@ +module playcmds + +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.core.playcmds +import freeflowuniverse.herolib.core.pathlib + +fn test_play_mdbook() { + mut summary_path := pathlib.get_file(path: '/tmp/mdbook_test/SUMMARY.md', create: true)! + summar_content := ' +- [Page number 1](fruits/apple.md) +- [fruit intro](fruits/intro.md) +- [rpc page](rpc/tfchain.md) +- [vegies](test_vegetables/tomato.md) +' + summary_path.write(summar_content)! + + mut p := pathlib.get_file(path: '/tmp/heroscript/do.hero', create: true)! + // script := " + // !!doctree.new + // name: 'tree1' + + // !!doctree.add + // name: 'tree1' + // url:'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/actions' + + // !!doctree.add + // name: 'tree1' + // url: 'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/fruits' + + // !!doctree.add + // name: 'tree1' + // url: 'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/rpc' + + // !!doctree.export + // name: 'tree1' + // path: '/tmp/export_tree1' + + // !!doctree.new + // name: 'tree2' + // fail_on_error: true + + // !!doctree.add + // name: 'tree2' + // url: 'https://github.com/freeflowuniverse/herolib/tree/development_doctree4/herolib/data/doctree/testdata/vegetables' + + // !!doctree.export + // name: 'tree2' + // path: '/tmp/export_tree2' + + // !!mdbook.export + // title:'ThreeFold Technology' + // name:'tech' + // summary_path:'${summary_path.path}' + // collections:'/tmp/export_tree1,/tmp/export_tree2' + // dest: '/tmp/mdbook_export' + // production:0 //means we put it in summary + // " + + s2 := " +!!doctree.new + name: 'info_tfgrid' + fail_on_error: false + +!!doctree.add + name:'info_tfgrid' + url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/main/collections' + + +!!doctree.export + name:'info_tfgrid' + path:'~/hero/var/collections/info_tfgrid' + + +!!mdbook.export + title:'ThreeFold Technology' + name:'tech' + summary_url:'https://git.ourworld.tf/tfgrid/info_tfgrid/src/branch/development/books/tech/SUMMARY.md' + collections:'~/hero/var/collections/info_tfgrid' + production:0 //means we put it in summary +" + p.write(s2)! + + mut plbook := playbook.new(path: '/tmp/heroscript')! + playcmds.play_mdbook(mut plbook)! +} diff --git a/lib/core/playcmds/play_ssh.v b/lib/core/playcmds/play_ssh.v new file mode 100644 index 00000000..52baed17 --- /dev/null +++ b/lib/core/playcmds/play_ssh.v @@ -0,0 +1,22 @@ +module playcmds + +import freeflowuniverse.herolib.osal.sshagent +import freeflowuniverse.herolib.core.playbook + +pub fn play_ssh(mut plbook playbook.PlayBook) ! { + mut agent := sshagent.new()! + for mut action in plbook.find(filter: 'sshagent.*')! { + mut p := action.params + match action.name { + 'key_add' { + name := p.get('name')! + privkey := p.get('privkey')! + agent.add(name, privkey)! + } + else { + return error('action name ${action.name} not supported') + } + } + action.done = true + } +} diff --git a/lib/core/playcmds/play_threefold.v b/lib/core/playcmds/play_threefold.v new file mode 100644 index 00000000..41c36925 --- /dev/null +++ b/lib/core/playcmds/play_threefold.v @@ -0,0 +1,63 @@ +module playcmds + +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.threefold.grid +import freeflowuniverse.herolib.threefold.tfrobot +import os + +pub fn play_threefold(mut plbook playbook.PlayBook) ! { + mut config_actions := plbook.find(filter: 'threefold.configure')! + + mnemonics_ := os.getenv_opt('TFGRID_MNEMONIC') or { '' } + mut ssh_key := os.getenv_opt('SSH_KEY') or { '' } + + tfrobot.configure('play', network: 'main', mnemonics: mnemonics_)! + + mut robot := tfrobot.get('play')! + + if config_actions.len > 1 { + return error('can only have 1 config action for threefold') + } else if config_actions.len == 1 { + mut a := config_actions[0] + mut p := a.params + mut network := p.get_default('network', 'main')! + mnemonics := p.get_default('mnemonics', '')! + ssh_key = p.get_default('ssh_key', '')! + + network = network.to_lower() + + // mnemonics string + // network string = 'main' + tfrobot.configure('play', network: network, mnemonics: mnemonics)! + + robot = tfrobot.get('play')! + + config_actions[0].done = true + } + cfg := robot.config()! + if cfg.mnemonics == '' { + return error('TFGRID_MNEMONIC should be specified as env variable') + } + + if ssh_key == '' { + return error('SSHKey should be specified as env variable') + } + + panic('implement') + + // for mut action in plbook.find(filter: 'threefold.deploy_vm')! { + // mut p := action.params + // deployment_name := p.get_default('deployment_name', 'deployment')! + // name := p.get_default('name', 'vm')! + // ssh_key := p.get_default('ssh_key', '')! + // cores := p.get_int_default('cores', 1)! + // memory := p.get_int_default('memory', 20)! + // panic("implement") + // action.done = true + // } + + // for mut action in plbook.find(filter: 'threefold.deploy_zdb')! { + // panic("implement") + // action.done = true + // } +} diff --git a/lib/core/playcmds/play_zola.v b/lib/core/playcmds/play_zola.v new file mode 100644 index 00000000..930d0e96 --- /dev/null +++ b/lib/core/playcmds/play_zola.v @@ -0,0 +1,246 @@ +module playcmds + +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.web.zola +import freeflowuniverse.herolib.core.playbook + +struct WebsiteItem { +mut: + name string + site ?&zola.ZolaSite +} + +pub fn play_zola(mut plbook playbook.PlayBook) ! { + // mut coderoot := '' + mut buildroot := '' + mut publishroot := '' + mut install := true + mut reset := false + + wsactions := plbook.find(filter: 'website.')! + if wsactions.len == 0 { + return + } + + mut config_actions := plbook.find(filter: 'websites:configure')! + if config_actions.len > 1 { + return error('can only have 1 config action for websites') + } else if config_actions.len == 1 { + mut p := config_actions[0].params + buildroot = p.get_default('buildroot', '')! + publishroot = p.get_default('publishroot', '')! + // coderoot = p.get_default('coderoot', '')! + install = p.get_default_true('install') + reset = p.get_default_false('reset') + config_actions[0].done = true + } + mut websites := zola.new( + path_build: buildroot + path_publish: publishroot + install: install + reset: reset + )! + + mut ws := WebsiteItem{} + + for mut action in plbook.find(filter: 'website.')! { + if action.name == 'define' { + console.print_debug('website.define') + mut p := action.params + ws.name = p.get('name')! + title := p.get_default('title', '')! + description := p.get_default('description', '')! + ws.site = websites.new(name: ws.name, title: title, description: description)! + } else if action.name == 'template_add' { + console.print_debug('website.template_add') + mut p := action.params + url := p.get_default('url', '')! + mut site_ := ws.site or { + return error("can't find website for template_add, should have been defined before with !!website.define") + } + + site_.template_add(url: url)! + } else if action.name == 'content_add' { + console.print_debug('website.content_add') + mut p := action.params + url := p.get_default('url', '')! + mut site_ := ws.site or { + return error("can't find website for content_add, should have been defined before with !!website.define") + } + + site_.content_add(url: url)! + } else if action.name == 'doctree_add' { + console.print_debug('website.doctree_add') + mut p := action.params + url := p.get_default('url', '')! + pull := p.get_default_false('pull') + mut site_ := ws.site or { + return error("can't find website for doctree_add, should have been defined before with !!website.define") + } + + site_.doctree_add(url: url, pull: pull)! + } else if action.name == 'post_add' { + console.print_debug('website.post_add') + mut p := action.params + name := p.get_default('name', '')! + collection := p.get_default('collection', '')! + file := p.get_default('file', '')! + page := p.get_default('page', '')! + pointer := p.get_default('pointer', '')! + mut site_ := ws.site or { + return error("can't find website for doctree_add, should have been defined before with !!website.define") + } + + site_.post_add(name: name, collection: collection, file: file, pointer: pointer)! + } else if action.name == 'blog_add' { + console.print_debug('website.blog_add') + mut p := action.params + name := p.get_default('name', '')! + collection := p.get_default('collection', '')! + file := p.get_default('file', '')! + page := p.get_default('page', '')! + pointer := p.get_default('pointer', '')! + mut site_ := ws.site or { + return error("can't find website for doctree_add, should have been defined before with !!website.define") + } + + site_.blog_add(name: name)! + } else if action.name == 'person_add' { + console.print_debug('website.person_add') + mut p := action.params + name := p.get_default('name', '')! + page := p.get_default('page', '')! + collection := p.get_default('collection', '')! + file := p.get_default('file', '')! + pointer := p.get_default('pointer', '')! + mut site_ := ws.site or { + return error("can't find website for doctree_add, should have been defined before with !!website.define") + } + + site_.person_add( + name: name + collection: collection + file: file + page: page + pointer: pointer + )! + } else if action.name == 'people_add' { + console.print_debug('website.people_add') + mut p := action.params + name := p.get_default('name', '')! + description := p.get_default('description', '')! + sort_by_ := p.get_default('sort_by', '')! + mut site_ := ws.site or { + return error("can't find website for people_add, should have been defined before with !!website.define") + } + + sort_by := zola.SortBy.from(sort_by_)! + site_.people_add( + name: name + title: p.get_default('title', '')! + sort_by: sort_by + description: description + )! + } else if action.name == 'blog_add' { + console.print_debug('website.blog_add') + mut p := action.params + name := p.get_default('name', '')! + description := p.get_default('description', '')! + sort_by_ := p.get_default('sort_by', '')! + mut site_ := ws.site or { + return error("can't find website for people_add, should have been defined before with !!website.define") + } + + sort_by := zola.SortBy.from(sort_by_)! + site_.blog_add( + name: name + title: p.get_default('title', '')! + sort_by: sort_by + description: description + )! + } else if action.name == 'news_add' { + console.print_debug('website.news_add') + mut p := action.params + name := p.get_default('name', '')! + collection := p.get_default('collection', '')! + pointer := p.get_default('pointer', '')! + file := p.get_default('file', '')! + mut site_ := ws.site or { + return error("can't find website for news_add, should have been defined before with !!website.define") + } + + site_.article_add(name: name, collection: collection, file: file, pointer: pointer)! + } else if action.name == 'header_add' { + console.print_debug('website.header_add') + mut p := action.params + template := p.get_default('template', '')! + logo := p.get_default('logo', '')! + mut site_ := ws.site or { + return error("can't find website for doctree_add, should have been defined before with !!website.define") + } + + site_.header_add(template: template, logo: logo)! + } else if action.name == 'header_link_add' { + console.print_debug('website.header_link_add') + mut p := action.params + page := p.get_default('page', '')! + label := p.get_default('label', '')! + mut site_ := ws.site or { + return error("can't find website for header_link_add, should have been defined before with !!website.define") + } + + site_.header_link_add(page: page, label: label)! + } else if action.name == 'footer_add' { + console.print_debug('website.footer_add') + mut p := action.params + template := p.get_default('template', '')! + mut site_ := ws.site or { + return error("can't find website for doctree_add, should have been defined before with !!website.define") + } + + site_.footer_add(template: template)! + } else if action.name == 'page_add' { + console.print_debug('website.page_add') + mut p := action.params + name := p.get_default('name', '')! + collection := p.get_default('collection', '')! + file := p.get_default('file', '')! + homepage := p.get_default_false('homepage') + mut site_ := ws.site or { + return error("can't find website for doctree_add, should have been defined before with !!website.define") + } + + site_.page_add(name: name, collection: collection, file: file, homepage: homepage)! + + // }else if action.name=="pull"{ + // mut site_:=ws.site or { return error("can't find website for pull, should have been defined before with !!website.define")} + // site_.pull()! + } else if action.name == 'section_add' { + console.print_debug('website.section_add') + // mut p := action.params + // name := p.get_default('name', '')! + // // collection := p.get_default('collection', '')! + // // file := p.get_default('file', '')! + // // homepage := p.get_default_false('homepage') + // mut site_ := ws.site or { + // return error("can't find website for doctree_add, should have been defined before with !!website.define") + // } + + // site_.add_section(name: name)! + + // }else if action.name=="pull"{ + // mut site_:=ws.site or { return error("can't find website for pull, should have been defined before with !!website.define")} + // site_.pull()! + } else if action.name == 'generate' { + mut site_ := ws.site or { + return error("can't find website for generate, should have been defined before with !!website.define") + } + + site_.generate()! + // site_.serve()! + } else { + return error("Cannot find right action for website. Found '${action.name}' which is a non understood action for !!website.") + } + action.done = true + } +} diff --git a/lib/core/playcmds/readme.md b/lib/core/playcmds/readme.md new file mode 100644 index 00000000..a1d4788b --- /dev/null +++ b/lib/core/playcmds/readme.md @@ -0,0 +1,28 @@ +# how to sue the playcmds + +```v +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.core.base + +mut s:=base.session_new( + coderoot:'/tmp/code' + interactive:true +)! + + +// path string +// text string +// git_url string +// git_pull bool +// git_branch string +// git_reset bool +// execute bool = true +// session ?&base.Session is optional + +mut plbook := playbook.new(text: "....",session:s) or { panic(err) } + + + + + +``` \ No newline at end of file diff --git a/lib/core/playcmds/scheduler.v b/lib/core/playcmds/scheduler.v new file mode 100644 index 00000000..ca4bfc9e --- /dev/null +++ b/lib/core/playcmds/scheduler.v @@ -0,0 +1,9 @@ +module playcmds + +import freeflowuniverse.herolib.installers.sysadmintools.daguserver + +pub fn scheduler(heroscript string) ! { + daguserver.play( + heroscript: heroscript + )! +} diff --git a/lib/core/playmacros/playmacros.v b/lib/core/playmacros/playmacros.v new file mode 100644 index 00000000..689b17e0 --- /dev/null +++ b/lib/core/playmacros/playmacros.v @@ -0,0 +1,38 @@ +module playmacros + +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.core.playbook +import freeflowuniverse.herolib.threefold.grid4.gridsimulator +import freeflowuniverse.herolib.threefold.grid4.farmingsimulator +import freeflowuniverse.herolib.biz.bizmodel +import freeflowuniverse.herolib.biz.spreadsheet + +pub fn play_actions(mut plbook playbook.PlayBook) ! { + console.print_green('play actions (simulators)') + farmingsimulator.play(mut plbook)! + gridsimulator.play(mut plbook)! + bizmodel.play(mut plbook)! +} + +pub fn play_macro(action playbook.Action) !string { + if action.actiontype != .macro { + panic('should always be a macro') + } + console.print_green('macro: ${action.actor}:${action.name}') + if action.actor == 'sheet' || action.actor == 'spreadsheet' { + return spreadsheet.playmacro(action) or { + return 'Macro error: ${action.actor}:${action.name}\n${err}' + } + } else if action.actor == 'tfgridsimulation_farming' { + return farmingsimulator.playmacro(action) or { + return 'Macro error: ${action.actor}:${action.name}\n${err}' + } + } else if action.actor == 'bizmodel' { + return bizmodel.playmacro(action) or { + return 'Macro error: ${action.actor}:${action.name}\n${err}' + } + } else { + return "Macro error, Couldn't find macro: '${action.actor}:${action.name}'" + } + return '' +} diff --git a/research/globals/globals_example.vsh b/research/globals/globals_example.vsh new file mode 100755 index 00000000..5d6966e4 --- /dev/null +++ b/research/globals/globals_example.vsh @@ -0,0 +1,133 @@ +#!/usr/bin/env -S v -enable-globals run + +import rand +import os +import time + +__global ( + bizmodels shared map[string]BizModel +) + +pub struct SubItem { +pub mut: + name string + intlist []int + intstr []string +} + +pub struct BizModel { +pub mut: + name string + intlist []int + intstr []string + mymap map[string]SubItem +} + +pub fn biz_model_example(name string) BizModel { + mut biz_model := BizModel{ + name: name + intlist: []int{len: 10, init: rand.intn(100) or { 0 }} + intstr: []string{len: 10, init: rand.string(5)} + mymap: map[string]SubItem{} + } + + for i in 0 .. 100 { + sub_item := SubItem{ + name: 'SubItem ${i}' + intlist: []int{len: 5, init: rand.intn(50) or { 0 }} + intstr: []string{len: 5, init: rand.string(3)} + } + biz_model.mymap['item_${i}'] = sub_item + } + + return biz_model + +} + +pub fn get(name string) !BizModel { + rlock bizmodels { + if ! (name in bizmodels) { + return error("can't find bizmodel: ${name}") + } + return bizmodels[name] or { panic("bug") } + } + return error("bug") +} + + +pub fn getset(name string) BizModel { + lock bizmodels { + if ! (name in bizmodels) { + set(biz_model_example(name)) + } + return bizmodels[name] or { panic("bug") } + } + return BizModel{} //weird we need to do this +} + +pub fn set(bizmodel BizModel) { + + lock bizmodels { + bizmodels[bizmodel.name] = bizmodel + } + +} + +fn fill_biz_models(nr int) { + for i in 0 .. nr { + getset("bm${i}") + } + rlock bizmodels{ + //check we have enough bizmodels in mem + assert bizmodels.len == nr + } + +} + +fn get_memory_usage() i64 { + pid := os.getpid() + res := os.execute('ps -o rss= -p ${pid}') + return res.output.trim_space().i64() +} + +fn main() { + sw := time.new_stopwatch() + + nr:=100 + nr_new :=100000 //make small changes, the garbage collector should keep it clean + + fill_biz_models(nr) + + memory_usage_1 := get_memory_usage() + println('Memory usage after creating ${nr} BizModels: ${memory_usage_1} KB') + println('Time taken: ${sw.elapsed().milliseconds()} ms') + + for _ in 0 .. nr_new { + currentnr:=rand.intn(nr-1) or {panic(err)} + mut new_model:= get("bm${currentnr}")! + //new_model.intlist = new_model.intlist.map(it + rand.intn(10) or { 0 }) + new_model.intstr = new_model.intstr.map(it + rand.string(2)) + mut new_model2:= get("bm${currentnr}")! + assert new_model2.intstr != new_model.intstr //should be different because was not a reference + set(new_model) + mut new_model3:= get("bm${currentnr}")! + assert new_model3.intstr == new_model.intstr //should be different because was not a reference + + } + + rlock bizmodels{ + //check we have enough bizmodels in mem + assert bizmodels.len == nr + } + + println("wait 1 sec") + //lets make sure garbage collector works + time.sleep(1 * time.second) + + + memory_usage_2 := get_memory_usage() + println('Memory usage after creating ${nr_new} random BizModels: ${memory_usage_2} KB') + println('Time taken: ${sw.elapsed().milliseconds()} ms') + + println("QUESTION: why does memory go up?, we didn't add to the memory should have been equal...") +} \ No newline at end of file diff --git a/research/globals/globals_example_inplace.vsh b/research/globals/globals_example_inplace.vsh new file mode 100755 index 00000000..0780f098 --- /dev/null +++ b/research/globals/globals_example_inplace.vsh @@ -0,0 +1,125 @@ +#!/usr/bin/env -S v -enable-globals run + +import rand +import os +import time + +__global ( + bizmodels shared map[string]BizModel +) + +@[heap] +pub struct SubItem { +pub mut: + name string + intlist []int + intstr []string +} + +pub struct BizModel { +pub mut: + name string + intlist []int + intstr []string + mymap map[string]SubItem +} + +pub fn biz_model_example(name string) BizModel { + mut biz_model := BizModel{ + name: name + intlist: []int{len: 10, init: rand.intn(100) or { 0 }} + intstr: []string{len: 10, init: rand.string(5)} + mymap: map[string]SubItem{} + } + + for i in 0 .. 100 { + sub_item := SubItem{ + name: 'SubItem ${i}' + intlist: []int{len: 5, init: rand.intn(50) or { 0 }} + intstr: []string{len: 5, init: rand.string(3)} + } + biz_model.mymap['item_${i}'] = sub_item + } + + return biz_model + +} + +pub fn get(name string) !BizModel { + rlock bizmodels { + if ! (name in bizmodels) { + return error("can't find bizmodel: ${name}") + } + return bizmodels[name] or { panic("bug") } + } + return error("bug") +} + + +pub fn getset(name string) BizModel { + lock bizmodels { + if ! (name in bizmodels) { + set(biz_model_example(name)) + } + return bizmodels[name] or { panic("bug") } + } + return BizModel{} //weird we need to do this +} + +pub fn set(bizmodel BizModel) { + lock bizmodels { + bizmodels[bizmodel.name] = bizmodel + } + +} + +fn fill_biz_models(nr int) { + for i in 0 .. nr { + getset("bm${i}") + } + rlock bizmodels{ + //check we have enough bizmodels in mem + assert bizmodels.len == nr + } + +} + +fn get_memory_usage() i64 { + pid := os.getpid() + res := os.execute('ps -o rss= -p ${pid}') + return res.output.trim_space().i64() +} + +fn main() { + sw := time.new_stopwatch() + + nr:=100 + nr_new :=100000 //make small changes, the garbage collector should keep it clean + + fill_biz_models(nr) + + println("wait 0.5 sec") + //lets make sure garbage collector works + time.sleep(0.5 * time.second) + + memory_usage_1 := get_memory_usage() + println('Memory usage after creating ${nr} BizModels: ${memory_usage_1} KB') + println('Time taken: ${sw.elapsed().milliseconds()} ms') + + for _ in 0 .. nr_new { + currentnr:=rand.intn(nr-1) or {panic(err)} + //println(currentnr) + set(biz_model_example("bm${currentnr}")) //will keep on overwriting + } + + println("wait 1 sec") + //lets make sure garbage collector works + time.sleep(1 * time.second) + + + memory_usage_2 := get_memory_usage() + println('Memory usage after creating ${nr_new} random BizModels: ${memory_usage_2} KB') + println('Time taken: ${sw.elapsed().milliseconds()} ms') + + println("QUESTION: this seems ok, memory doesn't go up much\nDon't understand why the globals_example does increase.") +} \ No newline at end of file diff --git a/research/globals/globals_example_reference.vsh b/research/globals/globals_example_reference.vsh new file mode 100755 index 00000000..b7f22333 --- /dev/null +++ b/research/globals/globals_example_reference.vsh @@ -0,0 +1,124 @@ +#!/usr/bin/env -S v -enable-globals run + +import rand +import os +import time + +__global ( + bizmodels shared map[string]&BizModel +) + +pub struct SubItem { +pub mut: + name string + intlist []int + intstr []string +} + +pub struct BizModel { +pub mut: + name string + intlist []int + intstr []string + mymap map[string]SubItem +} + +pub fn biz_model_example(name string) BizModel { + mut biz_model := BizModel{ + name: name + intlist: []int{len: 10, init: rand.intn(100) or { 0 }} + intstr: []string{len: 10, init: rand.string(5)} + mymap: map[string]SubItem{} + } + + for i in 0 .. 100 { + sub_item := SubItem{ + name: 'SubItem ${i}' + intlist: []int{len: 5, init: rand.intn(50) or { 0 }} + intstr: []string{len: 5, init: rand.string(3)} + } + biz_model.mymap['item_${i}'] = sub_item + } + + return biz_model + +} + +pub fn get(name string) !&BizModel { + rlock bizmodels { + if ! (name in bizmodels) { + return error("can't find bizmodel: ${name}") + } + return bizmodels[name] or { panic("bug") } + } + return error("bug") +} + + +pub fn getset(name string) &BizModel { + lock bizmodels { + if ! (name in bizmodels) { + //println("getset ${name}") + new(biz_model_example(name)) + } + return bizmodels[name] or { panic("bug") } + } + return &BizModel{} //weird we need to do this +} + +pub fn new(bizmodel BizModel) { + lock bizmodels { + bizmodels[bizmodel.name] = &bizmodel + } + +} + +fn fill_biz_models(nr int) { + for i in 0 .. nr { + getset("bm${i}") + } + rlock bizmodels{ + assert bizmodels.len == nr + } + +} + +fn get_memory_usage() i64 { + pid := os.getpid() + res := os.execute('ps -o rss= -p ${pid}') + return res.output.trim_space().i64() +} + +fn main() { + sw := time.new_stopwatch() + + nr:=100 + nr_new :=100000 //make small changes, the garbage collector should keep it clean + + fill_biz_models(nr) + + memory_usage_1 := get_memory_usage() + println('Memory usage after creating ${nr} BizModels: ${memory_usage_1} KB') + println('Time taken: ${sw.elapsed().milliseconds()} ms') + + for _ in 0 .. nr_new { + currentnr:=rand.intn(nr-1) or {panic(err)} + mut new_model:= get("bm${currentnr}")! + //new_model.intlist = new_model.intlist.map(it + rand.intn(10) or { 0 }) + new_model.intstr = new_model.intstr.map(it + rand.string(2)) + //set(new_model) //SHOULD NOT BE NEEDED TO SET BECAUSE IS REFERENCE + mut new_model2:= get("bm${currentnr}")! + assert new_model2.intstr == new_model.intstr //should be same because was a reference + + } + + println("wait 1 sec") + //lets make sure garbage collector works, didn't make a difference though + time.sleep(1 * time.second) + + memory_usage_2 := get_memory_usage() + println('Memory usage after creating ${nr_new} random BizModels: ${memory_usage_2} KB') + println('Time taken: ${sw.elapsed().milliseconds()} ms') + + println("QUESTION: why does memory go up?, we didn't add to the memory should have been equal... its also not lower compared to the references one") +} \ No newline at end of file