MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

Merge branch 'development_hetzner' of https://github.com/freeflowuniverse/herolib into development_hetzner

Browse Source
This commit is contained in:
despiegk
2025-02-07 12:55:21 +03:00
parent f8d675dcaf a0c253fa05
commit 65ec6ee1a3
55 changed files with 1544 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/core/openapi/gitea/gitea_client/factory.v
View File

@@ -1,7 +1,7 @@
module dagu
// import os
import freeflowuniverse.herolib.clients.httpconnection
import freeflowuniverse.herolib.core.httpconnection
import os
struct GiteaClient[T] {

8
examples/develop/ipapi/example.vsh Executable file
View File

@@ -0,0 +1,8 @@
#!/usr/bin/env -S v -n -w -gc none -no-retry-compilation -d use_openssl -enable-globals run
import freeflowuniverse.herolib.clients.ipapi
import os
mut ip_api_client := ipapi.get()!
info := ip_api_client.get_ip_info('37.27.132.46')!
println('info: ${info}')

8
examples/develop/wireguard/wg0.conf Normal file
View File

@@ -0,0 +1,8 @@
[Interface]
Address = 10.10.3.0/24
PrivateKey = wDewSiri8jlaGnUDN6SwK7QhN082U7gfX27YMGILvVA=
[Peer]
PublicKey = 2JEGJQ8FbajdFk0fFs/881H/D3FRjwlUxvNDZFxDeWQ=
AllowedIPs = 10.10.0.0/16, 100.64.0.0/16
PersistentKeepalive = 25
Endpoint = 185.206.122.31:3241

35
examples/develop/wireguard/wireguard.vsh Executable file
View File

@@ -0,0 +1,35 @@
#!/usr/bin/env -S v -n -w -gc none -no-retry-compilation -d use_openssl -enable-globals run
import freeflowuniverse.herolib.clients.wireguard
import freeflowuniverse.herolib.installers.net.wireguard as wireguard_installer
import time
import os
mut wg_installer := wireguard_installer.get()!
wg_installer.install()!
// Create Wireguard client
mut wg := wireguard.get()!
config_file_path := '${os.dir(@FILE)}/wg0.conf'
wg.start(config_file_path: config_file_path)!
println('${config_file_path} is started')
time.sleep(time.second * 2)
info := wg.show()!
println('info: ${info}')
config := wg.show_config(interface_name: 'wg0')!
println('config: ${config}')
private_key := wg.generate_private_key()!
println('private_key: ${private_key}')
public_key := wg.get_public_key(private_key: private_key)!
println('public_key: ${public_key}')
wg.down(config_file_path: config_file_path)!
println('${config_file_path} is down')
wg_installer.destroy()!

11
examples/installers/buildah.vsh Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env -S v -n -w -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run
import freeflowuniverse.herolib.installers.virt.buildah as buildah_installer
mut buildah := buildah_installer.get()!
// To install
buildah.install()!
// To remove
buildah.destroy()!

11
examples/installers/docker.vsh Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env -S v -n -w -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run
import freeflowuniverse.herolib.installers.virt.docker as docker_installer
mut docker := docker_installer.get()!
// To install
docker.install()!
// To remove
docker.destroy()!

47
examples/threefold/tfgrid3deployer/gw_over_wireguard/gw_over_wireguard.vsh Executable file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env -S v -gc none -no-retry-compilation -d use_openssl -enable-globals -cg run
//#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals -cg run
import freeflowuniverse.herolib.threefold.gridproxy
import freeflowuniverse.herolib.threefold.tfgrid3deployer
import freeflowuniverse.herolib.installers.threefold.griddriver
import os
import time
griddriver.install()!
v := tfgrid3deployer.get()!
println('cred: ${v}')
deployment_name := 'wireguard_dep_example'
mut deployment := tfgrid3deployer.new_deployment(deployment_name)!
deployment.configure_network(user_access_endpoints: 3)!
deployment.add_machine(
name: 'vm1'
cpu: 1
memory: 2
planetary: false
public_ip4: true
size: 10 // 10 gig
mycelium: tfgrid3deployer.Mycelium{}
)
deployment.deploy()!
vm1 := deployment.vm_get('vm1')!
println('vm1 info: ${vm1}')
user_access_configs := deployment.get_user_access_configs()
for config in user_access_configs {
println('config:\n------\n${config.print_wg_config()}\n------\n')
}
deployment.add_webname(
name: 'gwoverwg'
backend: 'http://${vm1.wireguard_ip}:8000'
use_wireguard_network: true
)
deployment.deploy()!
gw1 := deployment.webname_get('gwoverwg')!
println('gw info: ${gw1}')
// tfgrid3deployer.delete_deployment(deployment_name)!

0
examples/virt/docker/docker_dev_tools.vsh Normal file → Executable file
View File

0
examples/virt/docker/docker_init.vsh Normal file → Executable file
View File

0
examples/virt/docker/docker_registry.vsh Normal file → Executable file
View File

0
examples/virt/docker/presearch_docker.vsh Normal file → Executable file
View File

12
examples/virt/docker/tf_dashboard.vsh Normal file → Executable file
View File

@@ -17,25 +17,25 @@ recipe.add_run(cmd: 'npm i -g yarn')!
recipe.add_run(
cmd: '
git clone https://github.com/threefoldtech/tfgrid-sdk-ts.git /app
cd /app/packages/dashboard
cd /app/packages/playground
yarn install
yarn lerna run build --no-private
yarn workspace @threefold/dashboard build
yarn workspace @threefold/playground build
'
)!
recipe.add_run(
cmd: '
rm /etc/nginx/conf.d/default.conf
cp /app/packages/dashboard/nginx.conf /etc/nginx/conf.d
cp /app/packages/playground/nginx.conf /etc/nginx/conf.d
apk add --no-cache bash
chmod +x /app/packages/dashboard/scripts/build-env.sh
cp -r /app/packages/dashboard/dist /usr/share/nginx/html
chmod +x /app/packages/playground/scripts/build-env.sh
cp -r /app/packages/playground/dist /usr/share/nginx/html
'
)!
recipe.add_run(cmd: 'echo "daemon off;" >> /etc/nginx/nginx.conf')!
recipe.add_cmd(cmd: '/bin/bash -c /app/packages/dashboard/scripts/build-env.sh')!
recipe.add_cmd(cmd: '/bin/bash -c /app/packages/playground/scripts/build-env.sh')!
recipe.add_entrypoint(cmd: 'nginx')!
recipe.build(false)!

6
lib/builder/node_factory.v
View File

@@ -4,7 +4,10 @@ import freeflowuniverse.herolib.data.ipaddress
// get node connection to local machine
pub fn (mut bldr BuilderFactory) node_local() !&Node {
return bldr.node_new(name: 'localhost')
return bldr.node_new(
name: 'localhost'
ipaddr: '127.0.0.1'
)
}
// format ipaddr: localhost:7777 .
@@ -64,7 +67,6 @@ pub fn (mut bldr BuilderFactory) node_new(args_ NodeArguments) !&Node {
mut iadd := ipaddress.new(args.ipaddr)!
node.name = iadd.toname()!
}
wasincache := node.load()!
if wasincache && args.reload {

8
lib/clients/ipapi/.heroscript Normal file
View File

@@ -0,0 +1,8 @@
!!hero_code.generate_client
name:'ipapi'
classname:'IPApi'
singleton:0
default:1
hasconfig:1
reset:0

29
lib/clients/ipapi/client.v Normal file
View File

@@ -0,0 +1,29 @@
module ipapi
import json
pub struct IPInfo {
pub:
query string
status string
country string
country_code string @[json: 'countryCode']
region string
region_name string @[json: 'regionName']
city string
zip string
lat f32
lon f32
timezone string
isp string
org string
as string
}
pub fn (mut a IPApi) get_ip_info(ip string) !IPInfo {
mut conn := a.connection()!
res := conn.get_json(prefix: 'json/${ip}')!
info := json.decode(IPInfo, res)!
return info
}

102
lib/clients/ipapi/ipapi_factory_.v Normal file
View File

@@ -0,0 +1,102 @@
module ipapi
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui.console
__global (
ipapi_global map[string]&IPApi
ipapi_default string
)
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string
}
fn args_get(args_ ArgsGet) ArgsGet {
mut args := args_
if args.name == '' {
args.name = ipapi_default
}
if args.name == '' {
args.name = 'default'
}
return args
}
pub fn get(args_ ArgsGet) !&IPApi {
mut args := args_get(args_)
if args.name !in ipapi_global {
if args.name == 'default' {
if !config_exists(args) {
if default {
config_save(args)!
}
}
config_load(args)!
}
}
return ipapi_global[args.name] or {
println(ipapi_global)
panic('could not get config for ipapi with name:${args.name}')
}
}
fn config_exists(args_ ArgsGet) bool {
mut args := args_get(args_)
mut context := base.context() or { panic('bug') }
return context.hero_config_exists('ipapi', args.name)
}
fn config_load(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
mut heroscript := context.hero_config_get('ipapi', args.name)!
play(heroscript: heroscript)!
}
fn config_save(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
context.hero_config_set('ipapi', args.name, heroscript_default()!)!
}
fn set(o IPApi) ! {
mut o2 := obj_init(o)!
ipapi_global[o.name] = &o2
ipapi_default = o.name
}
@[params]
pub struct PlayArgs {
pub mut:
heroscript string // if filled in then plbook will be made out of it
plbook ?playbook.PlayBook
reset bool
}
pub fn play(args_ PlayArgs) ! {
mut args := args_
if args.heroscript == '' {
args.heroscript = heroscript_default()!
}
mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
mut install_actions := plbook.find(filter: 'ipapi.configure')!
if install_actions.len > 0 {
for install_action in install_actions {
mut p := install_action.params
cfg_play(p)!
}
}
}
// switch instance to be used for ipapi
pub fn switch(name string) {
ipapi_default = name
}

58
lib/clients/ipapi/ipapi_model.v Normal file
View File

@@ -0,0 +1,58 @@
module ipapi
import freeflowuniverse.herolib.data.paramsparser
import freeflowuniverse.herolib.core.httpconnection
import os
pub const version = '1.14.3'
const singleton = false
const default = true
// TODO: THIS IS EXAMPLE CODE AND NEEDS TO BE CHANGED IN LINE TO STRUCT BELOW, IS STRUCTURED AS HEROSCRIPT
pub fn heroscript_default() !string {
heroscript := "
!!ipapi.configure
name:'default'
"
return heroscript
}
// THIS THE THE SOURCE OF THE INFORMATION OF THIS FILE, HERE WE HAVE THE CONFIG OBJECT CONFIGURED AND MODELLED
@[heap]
pub struct IPApi {
pub mut:
name string = 'default'
conn ?&httpconnection.HTTPConnection @[str: skip]
}
fn cfg_play(p paramsparser.Params) ! {
// THIS IS EXAMPLE CODE AND NEEDS TO BE CHANGED IN LINE WITH struct above
mut mycfg := IPApi{
name: p.get_default('name', 'default')!
}
set(mycfg)!
}
fn obj_init(obj_ IPApi) !IPApi {
// never call get here, only thing we can do here is work on object itself
mut obj := obj_
return obj
}
pub fn (mut client IPApi) connection() !&httpconnection.HTTPConnection {
mut c := client.conn or {
mut c2 := httpconnection.new(
name: 'ipapi_${client.name}'
url: 'http://ip-api.com'
cache: false
retry: 20
)!
c2
}
client.conn = c
return c
}

30
lib/clients/ipapi/readme.md Normal file
View File

@@ -0,0 +1,30 @@
# ipapi
To get started
```vlang
import freeflowuniverse.herolib.clients. ipapi
mut client:= ipapi.get()!
client...
```
## example heroscript
```hero
!!ipapi.configure
secret: '...'
host: 'localhost'
port: 8888
```

8
lib/clients/wireguard/.heroscript Normal file
View File

@@ -0,0 +1,8 @@
!!hero_code.generate_client
name:'wireguard'
classname:'WireGuard'
singleton:0
default:1
hasconfig:1
reset:0

114
lib/clients/wireguard/client.v Normal file
View File

@@ -0,0 +1,114 @@
module wireguard
import os
pub struct WGPeer {
pub mut:
endpoint string
allowed_ips string
latest_handshake string
transfer string
persistent_keepalive string
}
pub struct WGInterface {
pub mut:
name string
public_key string
listening_port int
}
pub struct WGInfo {
pub mut:
interface_ WGInterface
peers map[string]WGPeer
}
pub struct WGShow {
pub mut:
configs map[string]WGInfo
}
pub fn (wg WireGuard) show() !WGShow {
cmd := 'sudo wg show'
res := os.execute(cmd)
if res.exit_code != 0 {
return error('failed to execute show command due to: ${res.output}')
}
return wg.parse_show_command_output(res.output)
}
@[params]
pub struct ShowConfigArgs {
pub:
interface_name string @[required]
}
pub fn (wg WireGuard) show_config(args ShowConfigArgs) !WGInfo {
configs := wg.show()!.configs
config := configs[args.interface_name] or {
return error('key ${args.interface_name} does not exists.')
}
return config
}
@[params]
pub struct StartArgs {
pub:
config_file_path string @[required]
}
pub fn (wg WireGuard) start(args StartArgs) ! {
if !os.exists(args.config_file_path) {
return error('File ${args.config_file_path} does not exists.')
}
cmd := 'sudo wg-quick up ${args.config_file_path}'
res := os.execute(cmd)
if res.exit_code != 0 {
return error('failed to execute start command due to: ${res.output}')
}
}
@[params]
pub struct DownArgs {
pub:
config_file_path string @[required]
}
pub fn (wg WireGuard) down(args DownArgs) ! {
if !os.exists(args.config_file_path) {
return error('File ${args.config_file_path} does not exists.')
}
cmd := 'sudo wg-quick down ${args.config_file_path}'
res := os.execute(cmd)
if res.exit_code != 0 {
return error('failed to execute down command due to: ${res.output}')
}
}
pub fn (wg WireGuard) generate_private_key() !string {
cmd := 'wg genkey'
res := os.execute(cmd)
if res.exit_code != 0 {
return error('failed to execute genkey command due to: ${res.output}')
}
return res.output.trim_space()
}
@[params]
pub struct GetPublicKeyArgs {
pub:
private_key string @[required]
}
pub fn (wg WireGuard) get_public_key(args GetPublicKeyArgs) !string {
cmd := 'echo ${args.private_key} | wg pubkey'
res := os.execute(cmd)
if res.exit_code != 0 {
return error('failed to execute pubkey command due to: ${res.output}')
}
return res.output.trim_space()
}

30
lib/clients/wireguard/readme.md Normal file
View File

@@ -0,0 +1,30 @@
# wireguard
To get started
```vlang
import freeflowuniverse.herolib.clients. wireguard
mut client:= wireguard.get()!
client...
```
## example heroscript
```hero
!!wireguard.configure
secret: '...'
host: 'localhost'
port: 8888
```

72
lib/clients/wireguard/utils.v Normal file
View File

@@ -0,0 +1,72 @@
module wireguard
fn (wg WireGuard) parse_show_command_output(res string) !WGShow {
mut configs := map[string]WGInfo{}
mut lines := res.split('\n')
mut current_interface := ''
mut current_peers := map[string]WGPeer{}
mut iface := WGInterface{}
mut peer_key := ''
for line in lines {
mut parts := line.trim_space().split(': ')
if parts.len < 2 {
continue
}
key := parts[0]
value := parts[1]
if key.starts_with('interface') {
if current_interface != '' {
configs[current_interface] = WGInfo{
interface_: iface
peers: current_peers.clone()
}
current_peers.clear()
}
current_interface = value
iface = WGInterface{
name: current_interface
public_key: ''
listening_port: 0
}
} else if key == 'public key' {
iface.public_key = value
} else if key == 'listening port' {
iface.listening_port = value.int()
} else if key.starts_with('peer') {
peer_key = value
mut peer := WGPeer{
endpoint: ''
allowed_ips: ''
latest_handshake: ''
transfer: ''
persistent_keepalive: ''
}
current_peers[peer_key] = peer
} else if key == 'endpoint' {
current_peers[peer_key].endpoint = value
} else if key == 'allowed ips' {
current_peers[peer_key].allowed_ips = value
} else if key == 'latest handshake' {
current_peers[peer_key].latest_handshake = value
} else if key == 'transfer' {
current_peers[peer_key].transfer = value
} else if key == 'persistent keepalive' {
current_peers[peer_key].persistent_keepalive = value
}
}
if current_interface != '' {
configs[current_interface] = WGInfo{
interface_: iface
peers: current_peers.clone()
}
}
return WGShow{
configs: configs
}
}

102
lib/clients/wireguard/wireguard_factory_.v Normal file
View File

@@ -0,0 +1,102 @@
module wireguard
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
__global (
wireguard_global map[string]&WireGuard
wireguard_default string
)
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string
}
fn args_get(args_ ArgsGet) ArgsGet {
mut args := args_
if args.name == '' {
args.name = wireguard_default
}
if args.name == '' {
args.name = 'wireguard'
}
return args
}
pub fn get(args_ ArgsGet) !&WireGuard {
mut args := args_get(args_)
if args.name !in wireguard_global {
if args.name == 'wireguard' {
if !config_exists(args) {
if default {
println('When saving')
config_save(args)!
}
}
config_load(args)!
}
}
return wireguard_global[args.name] or {
println(wireguard_global)
panic('could not get config for wireguard with name:${args.name}')
}
}
fn config_exists(args_ ArgsGet) bool {
mut args := args_get(args_)
mut context := base.context() or { panic('bug') }
return context.hero_config_exists('wireguard', args.name)
}
fn config_load(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
mut heroscript := context.hero_config_get('wireguard', args.name)!
play(heroscript: heroscript)!
}
fn config_save(args_ ArgsGet) ! {
mut args := args_get(args_)
mut context := base.context()!
context.hero_config_set('wireguard', args.name, heroscript_default()!)!
}
fn set(o WireGuard) ! {
mut o2 := obj_init(o)!
wireguard_global[o.name] = &o2
wireguard_default = o.name
}
@[params]
pub struct PlayArgs {
pub mut:
heroscript string // if filled in then plbook will be made out of it
plbook ?playbook.PlayBook
reset bool
}
pub fn play(args_ PlayArgs) ! {
mut args := args_
if args.heroscript == '' {
args.heroscript = heroscript_default()!
}
mut plbook := args.plbook or { playbook.new(text: args.heroscript)! }
mut install_actions := plbook.find(filter: 'wireguard.configure')!
if install_actions.len > 0 {
for install_action in install_actions {
mut p := install_action.params
cfg_play(p)!
}
}
}
// switch instance to be used for wireguard
pub fn switch(name string) {
wireguard_default = name
}

38
lib/clients/wireguard/wireguard_model.v Normal file
View File

@@ -0,0 +1,38 @@
module wireguard
import freeflowuniverse.herolib.data.paramsparser
pub const version = '1.14.3'
const singleton = false
const default = true
// TODO: THIS IS EXAMPLE CODE AND NEEDS TO BE CHANGED IN LINE TO STRUCT BELOW, IS STRUCTURED AS HEROSCRIPT
pub fn heroscript_default() !string {
heroscript := "
!!wireguard.configure
name:'wireguard'
"
return heroscript
}
// THIS THE THE SOURCE OF THE INFORMATION OF THIS FILE, HERE WE HAVE THE CONFIG OBJECT CONFIGURED AND MODELLED
@[heap]
pub struct WireGuard {
pub mut:
name string = 'wireguard'
}
fn cfg_play(p paramsparser.Params) ! {
// THIS IS EXAMPLE CODE AND NEEDS TO BE CHANGED IN LINE WITH struct above
mut mycfg := WireGuard{
name: p.get_default('name', 'wireguard')!
}
set(mycfg)!
}
fn obj_init(obj_ WireGuard) !WireGuard {
// never call get here, only thing we can do here is work on object itself
mut obj := obj_
return obj
}

2
lib/crypt/openssl/generate.v
View File

@@ -5,6 +5,7 @@ import json
@[params]
pub struct OpenSSLGenerateArgs {
pub:
name string = 'default'
domain string = 'myregistry.domain.com'
reset bool
@@ -22,6 +23,7 @@ pub fn (mut ossl OpenSSL) generate(args OpenSSLGenerateArgs) !OpenSSLKey {
'
mut b := builder.new()!
println('b: ${b}')
mut node := b.node_local()!
node.exec(cmd: cmd)!

2
lib/installers/infra/coredns/cdns_install.v
View File

@@ -4,7 +4,7 @@ import freeflowuniverse.herolib.osal
import freeflowuniverse.herolib.osal.screen
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.core.texttools
import freeflowuniverse.herolib.clients.httpconnection
import freeflowuniverse.herolib.core.httpconnection
import os
@[params]

13
lib/installers/net/wireguard/.heroscript Normal file
View File

@@ -0,0 +1,13 @@
!!hero_code.generate_installer
name:'wireguard_installer'
classname:'WireGuard'
singleton:0
templates:0
default:1
title:''
supported_platforms:''
reset:0
startupmanager:0
hasconfig:0
build:0

44
lib/installers/net/wireguard/readme.md Normal file
View File

@@ -0,0 +1,44 @@
# wireguard
To get started
```vlang
import freeflowuniverse.herolib.installers.something.wireguard as wireguard_installer
heroscript:="
!!wireguard.configure name:'test'
password: '1234'
port: 7701
!!wireguard.start name:'test' reset:1
"
wireguard_installer.play(heroscript=heroscript)!
//or we can call the default and do a start with reset
//mut installer:= wireguard_installer.get()!
//installer.start(reset:true)!
```
## example heroscript
```hero
!!wireguard.configure
homedir: '/home/user/wireguard'
username: 'admin'
password: 'secretpassword'
title: 'Some Title'
host: 'localhost'
port: 8888
```

59
lib/installers/net/wireguard/wireguard_actions.v Normal file
View File

@@ -0,0 +1,59 @@
module wireguard
import freeflowuniverse.herolib.osal
import freeflowuniverse.herolib.installers.ulist
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.core
//////////////////// following actions are not specific to instance of the object
// checks if a certain version or above is installed
fn installed() !bool {
osal.execute_silent('wg --version') or { return false }
return true
}
// get the Upload List of the files
fn ulist_get() !ulist.UList {
return ulist.UList{}
}
// uploads to S3 server if configured
fn upload() ! {
}
fn install() ! {
console.print_header('install wireguard')
cmd := match core.platform()! {
.ubuntu {
'sudo apt install -y wireguard'
}
.osx {
'sudo brew install -y wireguard-tools'
}
else {
return error('unsupported platfrom ${core.platform()!}')
}
}
osal.execute_stdout(cmd)!
}
fn destroy() ! {
console.print_header('uninstall wireguard')
cmd := match core.platform()! {
.ubuntu {
'sudo apt remove -y wireguard wireguard-tools'
}
.osx {
'sudo brew uninstall -y wireguard-tools'
}
else {
return error('unsupported platform ${core.platform()!}')
}
}
osal.execute_stdout(cmd)!
}

74
lib/installers/net/wireguard/wireguard_factory_.v Normal file
View File

@@ -0,0 +1,74 @@
module wireguard
import freeflowuniverse.herolib.core.base
import freeflowuniverse.herolib.core.playbook
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.sysadmin.startupmanager
import freeflowuniverse.herolib.osal.zinit
import time
__global (
wireguard_installer_global map[string]&WireGuard
wireguard_installer_default string
)
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string
}
pub fn get(args_ ArgsGet) !&WireGuard {
return &WireGuard{}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////# LIVE CYCLE MANAGEMENT FOR INSTALLERS ///////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////
fn startupmanager_get(cat zinit.StartupManagerType) !startupmanager.StartupManager {
// unknown
// screen
// zinit
// tmux
// systemd
match cat {
.zinit {
console.print_debug('startupmanager: zinit')
return startupmanager.get(cat: .zinit)!
}
.systemd {
console.print_debug('startupmanager: systemd')
return startupmanager.get(cat: .systemd)!
}
else {
console.print_debug('startupmanager: auto')
return startupmanager.get()!
}
}
}
@[params]
pub struct InstallArgs {
pub mut:
reset bool
}
pub fn (mut self WireGuard) install(args InstallArgs) ! {
switch(self.name)
if args.reset || (!installed()!) {
install()!
}
}
pub fn (mut self WireGuard) destroy() ! {
switch(self.name)
destroy()!
}
// switch instance to be used for wireguard_installer
pub fn switch(name string) {
wireguard_installer_default = name
}

23
lib/installers/net/wireguard/wireguard_model.v Normal file
View File

@@ -0,0 +1,23 @@
module wireguard
pub const version = '1.14.3'
const singleton = false
const default = true
// THIS THE THE SOURCE OF THE INFORMATION OF THIS FILE, HERE WE HAVE THE CONFIG OBJECT CONFIGURED AND MODELLED
@[heap]
pub struct WireGuard {
pub mut:
name string = 'default'
}
fn obj_init(obj_ WireGuard) !WireGuard {
// never call get here, only thing we can do here is work on object itself
mut obj := obj_
return obj
}
// called before start if done
fn configure() ! {
// mut installer := get()!
}

3
lib/installers/virt/buildah/.heroscript
View File

@@ -1,4 +1,3 @@
!!hero_code.generate_installer
name:'buildah'
classname:'BuildahInstaller'
@@ -10,4 +9,4 @@
reset:0
startupmanager:0
hasconfig:0
build:1
build:0

70
lib/installers/virt/buildah/buildah_actions.v
View File

@@ -2,59 +2,26 @@ module buildah
import freeflowuniverse.herolib.osal
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.core.texttools
import freeflowuniverse.herolib.installers.ulist
import freeflowuniverse.herolib.installers.lang.golang
import os
import freeflowuniverse.herolib.core
// checks if a certain version or above is installed
fn installed_() !bool {
res := os.execute('${osal.profile_path_source_and()!} buildah -v')
if res.exit_code != 0 {
return false
}
r := res.output.split_into_lines().filter(it.trim_space().len > 0)
if r.len != 1 {
return error("couldn't parse herocontainers version, expected 'buildah -v' on 1 row.\n${res.output}")
}
v := texttools.version(r[0].all_after('version').all_before('(').replace('-dev', ''))
if texttools.version(version) == v {
return true
}
return false
fn installed() !bool {
osal.execute_silent('buildah -v') or { return false }
return true
}
fn install_() ! {
fn install() ! {
console.print_header('install buildah')
build()!
}
if core.platform()! != .ubuntu {
return error('Only ubuntu is supported for now')
}
fn build_() ! {
console.print_header('build buildah')
osal.package_install('runc,bats,btrfs-progs,git,go-md2man,libapparmor-dev,libglib2.0-dev,libgpgme11-dev,libseccomp-dev,libselinux1-dev,make,skopeo,libbtrfs-dev')!
mut g := golang.get()!
g.install()!
cmd := '
cd /tmp
rm -rf buildah
git clone https://github.com/containers/buildah
cd buildah
make SECURITYTAGS="apparmor seccomp"
'
cmd := 'sudo apt-get -y update && sudo apt-get -y install buildah'
osal.execute_stdout(cmd)!
// now copy to the default bin path
osal.cmd_add(
cmdname: 'buildah'
source: '/tmp/buildah/bin/buildah'
)!
osal.rm('
/tmp/buildah
')!
console.print_header('Buildah Installed Successfuly')
}
// get the Upload List of the files
@@ -64,17 +31,6 @@ fn ulist_get() !ulist.UList {
return ulist.UList{}
}
fn destroy_() ! {
osal.package_remove('
buildah
')!
// will remove all paths where go/bin is found
osal.profile_path_add_remove(paths2delete: 'go/bin')!
osal.rm('
buildah
/var/lib/buildah
/tmp/buildah
')!
fn destroy() ! {
osal.execute_stdout('sudo apt remove --purge -y buildah')!
}

52
lib/installers/virt/buildah/buildah_factory_.v
View File

@@ -14,29 +14,61 @@ __global (
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string
}
pub fn get(args_ ArgsGet) !&BuildahInstaller {
return &BuildahInstaller{}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////# LIVE CYCLE MANAGEMENT FOR INSTALLERS ///////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////
fn startupmanager_get(cat zinit.StartupManagerType) !startupmanager.StartupManager {
// unknown
// screen
// zinit
// tmux
// systemd
match cat {
.zinit {
console.print_debug('startupmanager: zinit')
return startupmanager.get(cat: .zinit)!
}
.systemd {
console.print_debug('startupmanager: systemd')
return startupmanager.get(cat: .systemd)!
}
else {
console.print_debug('startupmanager: auto')
return startupmanager.get()!
}
}
}
@[params]
pub struct InstallArgs {
pub mut:
reset bool
}
pub fn install(args InstallArgs) ! {
if args.reset {
destroy()!
}
if !(installed_()!) {
install_()!
pub fn (mut self BuildahInstaller) install(args InstallArgs) ! {
switch(self.name)
if args.reset || (!installed()!) {
install()!
}
}
pub fn destroy() ! {
destroy_()!
pub fn (mut self BuildahInstaller) destroy() ! {
switch(self.name)
destroy()!
}
pub fn build() ! {
build_()!
// switch instance to be used for buildah
pub fn switch(name string) {
buildah_default = name
}

6
lib/installers/virt/buildah/buildah_model.v
View File

@@ -1,18 +1,22 @@
module buildah
pub const version = '1.38.0'
const singleton = true
const default = true
// THIS THE THE SOURCE OF THE INFORMATION OF THIS FILE, HERE WE HAVE THE CONFIG OBJECT CONFIGURED AND MODELLED
@[heap]
pub struct BuildahInstaller {
pub mut:
name string = 'default'
}
fn obj_init(obj_ BuildahInstaller) !BuildahInstaller {
// never call get here, only thing we can do here is work on object itself
mut obj := obj_
return obj
}
// called before start if done
fn configure() ! {
// mut installer := get()!
}

13
lib/installers/virt/docker/.heroscript Normal file
View File

@@ -0,0 +1,13 @@
!!hero_code.generate_installer
name:'docker'
classname:'DockerInstaller'
singleton:0
templates:0
default:1
title:''
supported_platforms:''
reset:0
startupmanager:1
hasconfig:0
build:0

64
lib/installers/virt/docker/docker.v
View File

@@ -1,64 +0,0 @@
module docker
import freeflowuniverse.herolib.osal
import freeflowuniverse.herolib.installers.base
import freeflowuniverse.herolib.ui.console
// install docker will return true if it was already installed
pub fn install_() ! {
console.print_header('package install install docker')
if core.platform() != .ubuntu {
return error('only support ubuntu for now')
}
base.install()!
if !osal.done_exists('install_docker') && !osal.cmd_exists('docker') {
// osal.upgrade()!
osal.package_install('mc,wget,htop,apt-transport-https,ca-certificates,curl,software-properties-common')!
cmd := '
rm -f /usr/share/keyrings/docker-archive-keyring.gpg
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
apt update
apt-cache policy docker-ce
#systemctl status docker
'
osal.execute_silent(cmd)!
osal.package_install('docker-ce')!
check()!
osal.done_set('install_docker', 'OK')!
}
console.print_header('docker already done')
}
pub fn check() ! {
// todo: do a monitoring check to see if it works
cmd := '
# Check if docker command exists
if ! command -v docker &> /dev/null; then
echo "Error: Docker command-line tool is not installed."
exit 1
fi
# Check if Docker daemon is running
if ! pgrep -f "dockerd" &> /dev/null; then
echo "Error: Docker daemon is not running."
exit 1
fi
# Run the hello-world Docker container
output=$(docker run hello-world 2>&1)
if [[ "\$output" == *"Hello from Docker!"* ]]; then
echo "Docker is installed and running properly."
else
echo "Error: Failed to run the Docker hello-world container."
echo "Output: \$output"
exit 1
fi
'
r := osal.execute_silent(cmd)!
console.print_debug(r)
}

106
lib/installers/virt/docker/docker_actions.v Normal file
View File

@@ -0,0 +1,106 @@
module docker
import freeflowuniverse.herolib.core
import freeflowuniverse.herolib.osal
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.osal.zinit
import freeflowuniverse.herolib.installers.ulist
fn startupcmd() ![]zinit.ZProcessNewArgs {
mut res := []zinit.ZProcessNewArgs{}
res << zinit.ZProcessNewArgs{
name: 'docker'
cmd: 'dockerd'
}
return res
}
fn running() !bool {
console.print_header('Checking if Docker is running')
is_installed := installed() or {
return error('Cannot execute command docker, check if the docker is installed or call the `install()` method: ${err}')
}
if !is_installed {
return false
}
// Checking if the docker server responed
cmd := 'docker ps'
osal.execute_stdout(cmd) or { return false }
console.print_header('Docker is running')
return true
}
fn start_pre() ! {
}
fn start_post() ! {
}
fn stop_pre() ! {
}
fn stop_post() ! {
}
//////////////////// following actions are not specific to instance of the object
// checks if a certain version or above is installed
fn installed() !bool {
console.print_header('Checking if Docker is installed')
cmd := 'docker -v'
osal.execute_stdout(cmd) or { return false }
console.print_header('Docker is installed')
return true
}
// get the Upload List of the files
fn ulist_get() !ulist.UList {
return ulist.UList{}
}
// uploads to S3 server if configured
fn upload() ! {}
fn install() ! {
console.print_header('Installing Docker')
if core.platform()! != .ubuntu {
return error('only support ubuntu for now')
}
mut cmd := '
sudo apt-get update -y
sudo apt-get install -y ca-certificates curl
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "\$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update -y
'
osal.execute_stdout(cmd) or { return error('Cannot install docker due to: ${err}') }
cmd = 'sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin'
osal.execute_stdout(cmd) or { return error('Cannot install docker due to: ${err}') }
console.print_header('Docker installed sucessfully')
}
fn destroy() ! {
console.print_header('Removing Docker')
// Uninstall the Docker Engine, CLI, containerd, and Docker Compose packages:
mut cmd := 'sudo apt-get purge -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras'
osal.execute_stdout(cmd) or { return error('Cannot uninstall docker due to: ${err}') }
// Images, containers, volumes, or custom configuration files on your host aren't automatically removed. To delete all images, containers, and volumes:
cmd = 'sudo rm -rf /var/lib/docker && sudo rm -rf /var/lib/containerd'
osal.execute_stdout(cmd) or { return error('Cannot uninstall docker due to: ${err}') }
// Remove source list and keyrings
cmd = 'sudo rm /etc/apt/sources.list.d/docker.list && sudo rm /etc/apt/keyrings/docker.asc'
osal.execute_stdout(cmd) or { return error('Cannot uninstall docker due to: ${err}') }
console.print_header('Docker is removed')
}

147
lib/installers/virt/docker/docker_factory_.v Normal file
View File

@@ -0,0 +1,147 @@
module docker
import freeflowuniverse.herolib.ui.console
import freeflowuniverse.herolib.sysadmin.startupmanager
import freeflowuniverse.herolib.osal.zinit
import time
__global (
docker_global map[string]&DockerInstaller
docker_default string
)
/////////FACTORY
@[params]
pub struct ArgsGet {
pub mut:
name string
}
pub fn get(args_ ArgsGet) !&DockerInstaller {
return &DockerInstaller{}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////# LIVE CYCLE MANAGEMENT FOR INSTALLERS ///////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////
fn startupmanager_get(cat zinit.StartupManagerType) !startupmanager.StartupManager {
// unknown
// screen
// zinit
// tmux
// systemd
match cat {
.zinit {
console.print_debug('startupmanager: zinit')
return startupmanager.get(cat: .zinit)!
}
.systemd {
console.print_debug('startupmanager: systemd')
return startupmanager.get(cat: .systemd)!
}
else {
console.print_debug('startupmanager: auto')
return startupmanager.get()!
}
}
}
pub fn (mut self DockerInstaller) start() ! {
switch(self.name)
if self.running()! {
return
}
console.print_header('docker start')
if !installed()! {
install()!
}
configure()!
start_pre()!
for zprocess in startupcmd()! {
mut sm := startupmanager_get(zprocess.startuptype)!
console.print_debug('starting docker with ${zprocess.startuptype}...')
sm.new(zprocess)!
sm.start(zprocess.name)!
}
start_post()!
for _ in 0 .. 50 {
if self.running()! {
return
}
time.sleep(100 * time.millisecond)
}
return error('docker did not install properly.')
}
pub fn (mut self DockerInstaller) install_start(args InstallArgs) ! {
switch(self.name)
self.install(args)!
self.start()!
}
pub fn (mut self DockerInstaller) stop() ! {
switch(self.name)
stop_pre()!
for zprocess in startupcmd()! {
mut sm := startupmanager_get(zprocess.startuptype)!
sm.stop(zprocess.name)!
}
stop_post()!
}
pub fn (mut self DockerInstaller) restart() ! {
switch(self.name)
self.stop()!
self.start()!
}
pub fn (mut self DockerInstaller) running() !bool {
switch(self.name)
// walk over the generic processes, if not running return
for zprocess in startupcmd()! {
mut sm := startupmanager_get(zprocess.startuptype)!
r := sm.running(zprocess.name)!
if r == false {
return false
}
}
return running()!
}
@[params]
pub struct InstallArgs {
pub mut:
reset bool
}
pub fn (mut self DockerInstaller) install(args InstallArgs) ! {
switch(self.name)
if args.reset || (!installed()!) {
install()!
}
}
pub fn (mut self DockerInstaller) destroy() ! {
switch(self.name)
self.stop() or {}
destroy()!
}
// switch instance to be used for docker
pub fn switch(name string) {
docker_default = name
}

23
lib/installers/virt/docker/docker_model.v Normal file
View File

@@ -0,0 +1,23 @@
module docker
pub const version = '1.14.3'
const singleton = false
const default = true
// THIS THE THE SOURCE OF THE INFORMATION OF THIS FILE, HERE WE HAVE THE CONFIG OBJECT CONFIGURED AND MODELLED
@[heap]
pub struct DockerInstaller {
pub mut:
name string = 'default'
}
fn obj_init(obj_ DockerInstaller) !DockerInstaller {
// never call get here, only thing we can do here is work on object itself
mut obj := obj_
return obj
}
// called before start if done
fn configure() ! {
// mut installer := get()!
}

42
lib/installers/virt/docker/readme.md Normal file
View File

@@ -0,0 +1,42 @@
# docker
To get started
```vlang
import freeflowuniverse.herolib.installers.something.docker as docker_installer
heroscript:="
!!docker.configure name:'test'
password: '1234'
port: 7701
!!docker.start name:'test' reset:1
"
docker_installer.play(heroscript=heroscript)!
//or we can call the default and do a start with reset
//mut installer:= docker_installer.get()!
//installer.start(reset:true)!
```
## example heroscript
```hero
!!docker.configure
homedir: '/home/user/docker'
username: 'admin'
password: 'secretpassword'
title: 'Some Title'
host: 'localhost'
port: 8888
```

2
lib/threefold/rmb/rmb_client.v
View File

@@ -1,6 +1,6 @@
module rmb
// import freeflowuniverse.herolib.clients.httpconnection
// import freeflowuniverse.herolib.core.httpconnection
import freeflowuniverse.herolib.core.redisclient { RedisURL }
import os

24
lib/threefold/tfgrid3deployer/deployment.v
View File

@@ -1,7 +1,6 @@
module tfgrid3deployer
import freeflowuniverse.herolib.threefold.grid.models as grid_models
import freeflowuniverse.herolib.threefold.gridproxy.model as gridproxy_models
import freeflowuniverse.herolib.threefold.grid
import freeflowuniverse.herolib.ui.console
import compress.zlib
@@ -9,7 +8,6 @@ import encoding.hex
import x.crypto.chacha20
import crypto.sha256
import json
import rand
struct GridContracts {
pub mut:
@@ -51,7 +49,7 @@ pub fn new_deployment(name string) !TFDeployment {
kvstore := KVStoreFS{}
if _ := kvstore.get(name) {
return error('Deployment with the same name is already exist.')
return error('Deployment with the same name "${name}" already exists.')
}
deployer := get_deployer()!
@@ -114,6 +112,7 @@ pub fn (mut self TFDeployment) deploy() ! {
}
fn (mut self TFDeployment) set_nodes() ! {
// TODO: each request should run in a separate thread
for mut vm in self.vms {
if vm.node_id != 0 {
continue
@@ -136,6 +135,7 @@ fn (mut self TFDeployment) set_nodes() ! {
has_ipv6: if vm.requirements.public_ip6 { vm.requirements.public_ip6 } else { none }
status: 'up'
features: if vm.requirements.public_ip4 { ['zmachine'] } else { [] }
on_hetzner: vm.requirements.use_hetzner_node
)!
if nodes.len == 0 {
@@ -161,6 +161,7 @@ fn (mut self TFDeployment) set_nodes() ! {
healthy: true
node_id: zdb.requirements.node_id
available_for: u64(self.deployer.twin_id)
on_hetzner: zdb.requirements.use_hetzner_node
)!
if nodes.len == 0 {
@@ -184,6 +185,7 @@ fn (mut self TFDeployment) set_nodes() ! {
node_id: webname.requirements.node_id
available_for: u64(self.deployer.twin_id)
features: ['zmachine']
on_hetzner: webname.requirements.use_hetzner_node
)!
if nodes.len == 0 {
@@ -284,10 +286,10 @@ fn (mut self TFDeployment) finalize_deployment(setup DeploymentSetup) ! {
}
}
self.update_state(name_contracts_map, returned_deployments)!
self.update_state(setup, name_contracts_map, returned_deployments)!
}
fn (mut self TFDeployment) update_state(name_contracts_map map[string]u64, dls map[u32]&grid_models.Deployment) ! {
fn (mut self TFDeployment) update_state(setup DeploymentSetup, name_contracts_map map[string]u64, dls map[u32]&grid_models.Deployment) ! {
mut workloads := map[u32]map[string]&grid_models.Workload{}
for node_id, deployment in dls {
@@ -338,6 +340,10 @@ fn (mut self TFDeployment) update_state(name_contracts_map map[string]u64, dls m
wn.node_contract_id = dls[wn.node_id].contract_id
wn.name_contract_id = name_contracts_map[wn.requirements.name]
}
self.network.ip_range = setup.network_handler.ip_range
self.network.mycelium = setup.network_handler.mycelium
self.network.user_access_configs = setup.network_handler.user_access_configs.clone()
}
pub fn (mut self TFDeployment) vm_get(vm_name string) !VMachine {
@@ -512,3 +518,11 @@ pub fn (mut self TFDeployment) list_deployments() !map[u32]grid_models.Deploymen
return dls
}
pub fn (mut self TFDeployment) configure_network(req NetworkRequirements) ! {
self.network.requirements = req
}
pub fn (mut self TFDeployment) get_user_access_configs() []UserAccessConfig {
return self.network.user_access_configs
}

24
lib/threefold/tfgrid3deployer/deployment_setup.v
View File

@@ -32,14 +32,15 @@ fn new_deployment_setup(network_specs NetworkSpecs, vms []VMachine, zdbs []ZDB,
mut dls := DeploymentSetup{
deployer: deployer
network_handler: NetworkHandler{
deployer: deployer
network_name: network_specs.name
mycelium: network_specs.mycelium
ip_range: network_specs.ip_range
req: network_specs.requirements
deployer: deployer
mycelium: network_specs.mycelium
ip_range: network_specs.ip_range
user_access_configs: network_specs.user_access_configs.clone()
}
}
dls.setup_network_workloads(vms, old_deployments)!
dls.setup_network_workloads(vms, webnames, old_deployments)!
dls.setup_vm_workloads(vms)!
dls.setup_zdb_workloads(zdbs)!
dls.setup_webname_workloads(webnames)!
@@ -67,9 +68,9 @@ fn (mut self DeploymentSetup) match_versions(old_dls map[u32]grid_models.Deploym
// - st: Modified DeploymentSetup struct with network workloads set up
// Returns:
// - None
fn (mut st DeploymentSetup) setup_network_workloads(vms []VMachine, old_deployments map[u32]grid_models.Deployment) ! {
fn (mut st DeploymentSetup) setup_network_workloads(vms []VMachine, webnames []WebName, old_deployments map[u32]grid_models.Deployment) ! {
st.network_handler.load_network_state(old_deployments)!
st.network_handler.create_network(vms)!
st.network_handler.create_network(vms, webnames)!
data := st.network_handler.generate_workloads()!
for node_id, workload in data {
@@ -176,6 +177,11 @@ fn (mut self DeploymentSetup) setup_webname_workloads(webnames []WebName) ! {
tls_passthrough: req.tls_passthrough
backends: [req.backend]
name: gw_name
network: if wn.requirements.use_wireguard_network {
self.network_handler.req.name
} else {
none
}
}
self.workloads[wn.node_id] << gw.to_workload(
@@ -205,7 +211,7 @@ fn (mut self DeploymentSetup) set_zmachine_workload(vmachine VMachine, public_ip
network: grid_models.ZmachineNetwork{
interfaces: [
grid_models.ZNetworkInterface{
network: self.network_handler.network_name
network: self.network_handler.req.name
ip: if vmachine.wireguard_ip.len > 0 {
used_ip_octets[vmachine.node_id] << vmachine.wireguard_ip.all_after_last('.').u8()
vmachine.wireguard_ip
@@ -218,7 +224,7 @@ fn (mut self DeploymentSetup) set_zmachine_workload(vmachine VMachine, public_ip
planetary: vmachine.requirements.planetary
mycelium: if mycelium := vmachine.requirements.mycelium {
grid_models.MyceliumIP{
network: self.network_handler.network_name
network: self.network_handler.req.name
hex_seed: mycelium.hex_seed
}
} else {

122
lib/threefold/tfgrid3deployer/network.v
View File

@@ -1,23 +1,54 @@
module tfgrid3deployer
import freeflowuniverse.herolib.threefold.grid.models as grid_models
import freeflowuniverse.herolib.threefold.gridproxy
import freeflowuniverse.herolib.threefold.grid
import freeflowuniverse.herolib.ui.console
import json
import rand
// NetworkInfo struct to represent network details
@[params]
pub struct NetworkRequirements {
pub mut:
name string = 'net' + rand.string(5)
user_access_endpoints int
}
@[params]
pub struct NetworkSpecs {
pub mut:
name string = 'net' + rand.string(5)
ip_range string = '10.10.0.0/16'
mycelium string = rand.hex(64)
requirements NetworkRequirements
ip_range string = '10.10.0.0/16'
mycelium string = rand.hex(64)
user_access_configs []UserAccessConfig
}
pub struct UserAccessConfig {
pub:
ip string
secret_key string
public_key string
peer_public_key string
network_ip_range string
public_node_endpoint string
}
pub fn (c UserAccessConfig) print_wg_config() string {
return '[Interface]
Address = ${c.ip}
PrivateKey = ${c.secret_key}
[Peer]
PublicKey = ${c.peer_public_key}
AllowedIPs = ${c.network_ip_range}, 100.64.0.0/16
PersistentKeepalive = 25
Endpoint = ${c.public_node_endpoint}'
}
struct NetworkHandler {
mut:
network_name string
req NetworkRequirements
// network_name string
nodes []u32
ip_range string
wg_ports map[u32]u16
@@ -29,11 +60,14 @@ mut:
none_accessible_ip_ranges []string
mycelium string
// user_access_endopoints int
user_access_configs []UserAccessConfig
deployer &grid.Deployer @[skip; str: skip]
}
// TODO: maybe rename to fill_network or something similar
fn (mut self NetworkHandler) create_network(vmachines []VMachine) ! {
fn (mut self NetworkHandler) create_network(vmachines []VMachine, webnames []WebName) ! {
// Set nodes
self.nodes = []
@@ -43,9 +77,16 @@ fn (mut self NetworkHandler) create_network(vmachines []VMachine) ! {
}
}
for webname in webnames {
if webname.requirements.use_wireguard_network && !self.nodes.contains(webname.node_id) {
self.nodes << webname.node_id
}
}
console.print_header('Network nodes: ${self.nodes}.')
self.setup_wireguard_data()!
self.setup_access_node()!
self.setup_user_access()!
}
fn (mut self NetworkHandler) generate_workload(node_id u32, peers []grid_models.Peer, mycleium_hex_key string) !grid_models.Workload {
@@ -62,7 +103,7 @@ fn (mut self NetworkHandler) generate_workload(node_id u32, peers []grid_models.
}
return network_workload.to_workload(
name: self.network_name
name: self.req.name
description: 'VGridClient network workload'
)
}
@@ -70,10 +111,11 @@ fn (mut self NetworkHandler) generate_workload(node_id u32, peers []grid_models.
fn (mut self NetworkHandler) prepare_hidden_node_peers(node_id u32) ![]grid_models.Peer {
mut peers := []grid_models.Peer{}
if self.public_node != 0 {
ip_range_oct := self.ip_range.all_before('/').split('.')
peers << grid_models.Peer{
subnet: self.wg_subnet[self.public_node]
wireguard_public_key: self.wg_keys[self.public_node][1]
allowed_ips: [self.ip_range, '100.64.0.0/16']
allowed_ips: [self.ip_range, '100.64.${ip_range_oct[1]}.${ip_range_oct[2]}/24']
endpoint: '${self.endpoints[self.public_node]}:${self.wg_ports[self.public_node]}'
}
}
@@ -81,15 +123,7 @@ fn (mut self NetworkHandler) prepare_hidden_node_peers(node_id u32) ![]grid_mode
}
fn (mut self NetworkHandler) setup_access_node() ! {
// Case 1: Deployment on 28 which is hidden node
// - Setup access node
// Case 2: Deployment on 11 which is public node
// - Already have the access node
// Case 3: if the saved state has already public node.
// - Check the new deployment if its node is hidden take the saved one
// - if the access node is already set, that means we have set its values e.g. the wireguard port, keys
if self.hidden_nodes.len < 1 || self.nodes.len == 1 {
if self.req.user_access_endpoints == 0 && (self.hidden_nodes.len < 1 || self.nodes.len == 1) {
self.public_node = 0
return
}
@@ -144,8 +178,27 @@ fn (mut self NetworkHandler) setup_access_node() ! {
self.endpoints[self.public_node] = access_node.public_config.ipv4.split('/')[0]
}
fn (mut self NetworkHandler) setup_user_access() ! {
to_create_user_access := self.req.user_access_endpoints - self.user_access_configs.len
if to_create_user_access < 0 {
// TODO: support removing user access
return error('removing user access is not supported')
}
for i := 0; i < to_create_user_access; i++ {
wg_keys := self.deployer.client.generate_wg_priv_key()!
self.user_access_configs << UserAccessConfig{
ip: self.calculate_subnet()!
secret_key: wg_keys[0]
public_key: wg_keys[1]
peer_public_key: self.wg_keys[self.public_node][1]
public_node_endpoint: '${self.endpoints[self.public_node]}:${self.wg_ports[self.public_node]}'
network_ip_range: self.ip_range
}
}
}
fn (mut self NetworkHandler) setup_wireguard_data() ! {
// TODO: We need to set the extra node
console.print_header('Setting up network workload.')
self.hidden_nodes, self.none_accessible_ip_ranges = [], []
@@ -230,6 +283,17 @@ fn (mut self NetworkHandler) prepare_public_node_peers(node_id u32) ![]grid_mode
endpoint: ''
}
}
for user_access in self.user_access_configs {
routing_ip := wireguard_routing_ip(user_access.ip)
peers << grid_models.Peer{
subnet: user_access.ip
wireguard_public_key: user_access.public_key
allowed_ips: [user_access.ip, routing_ip]
endpoint: ''
}
}
}
return peers
@@ -237,10 +301,16 @@ fn (mut self NetworkHandler) prepare_public_node_peers(node_id u32) ![]grid_mode
fn (mut self NetworkHandler) calculate_subnet() !string {
mut parts := self.ip_range.split('/')[0].split('.')
user_access_subnets := self.user_access_configs.map(it.ip)
node_subnets := self.wg_subnet.values()
mut used_subnets := []string{}
used_subnets << node_subnets.clone()
used_subnets << user_access_subnets.clone()
for i := 2; i <= 255; i += 1 {
parts[2] = '${i}'
candidate := parts.join('.') + '/24'
if !self.wg_subnet.values().contains(candidate) {
if !used_subnets.contains(candidate) {
return candidate
}
}
@@ -269,7 +339,7 @@ fn (mut self NetworkHandler) load_network_state(dls map[u32]grid_models.Deployme
continue
}
self.network_name = network_name
self.req.name = network_name
self.nodes << node_id
self.ip_range = znet.ip_range
self.wg_ports[node_id] = znet.wireguard_listen_port
@@ -289,7 +359,11 @@ fn (mut self NetworkHandler) load_network_state(dls map[u32]grid_models.Deployme
}
for subnet, endpoint in subnet_to_endpoint {
node_id := subnet_node[subnet]
node_id := subnet_node[subnet] or {
// this maybe a user access, not a node
continue
}
if endpoint == '' {
self.hidden_nodes << node_id
continue
@@ -318,9 +392,3 @@ fn (mut self NetworkHandler) generate_workloads() !map[u32]grid_models.Workload
return workloads
}
fn (mut n NetworkHandler) remove_node(node_id u32) ! {
}
fn (mut n NetworkHandler) add_node() ! {
}

22
lib/threefold/tfgrid3deployer/utils.v
View File

@@ -42,16 +42,36 @@ fn get_mycelium() grid_models.Mycelium {
}
}
pub fn filter_nodes(filter gridproxy_models.NodeFilter) ![]gridproxy_models.Node {
@[params]
pub struct FilterNodesArgs {
gridproxy_models.NodeFilter
pub:
on_hetzner bool
}
pub fn filter_nodes(args FilterNodesArgs) ![]gridproxy_models.Node {
// Resolve the network configuration
net := resolve_network()!
// Create grid proxy client and retrieve the matching nodes
mut gp_client := gridproxy.new(net: net, cache: true)!
mut filter := args.NodeFilter
if args.on_hetzner {
filter.features << ['zmachine-light']
}
nodes := gp_client.get_nodes(filter)!
return nodes
}
// fn get_hetzner_node_ids(nodes []gridproxy_models.Node) ![]u64 {
// // get farm ids that are know to be hetzner's
// // if we need to iterate over all nodes, maybe we should use multi-threading
// panic('Not Implemented')
// return []
// }
fn convert_to_gigabytes(bytes u64) u64 {
return bytes * 1024 * 1024 * 1024
}

5
lib/threefold/tfgrid3deployer/vmachine.v
View File

@@ -26,7 +26,10 @@ pub mut:
flist string = 'https://hub.grid.tf/tf-official-vms/ubuntu-24.04-latest.flist'
entrypoint string = '/sbin/zinit init'
env map[string]string
nodes []u32 // if set will chose a node from the list to deploy on
// if set will chose a node from the list to deploy on
nodes []u32
// will deploy on one of hetzner nodes
use_hetzner_node bool
}
// MachineModel struct to represent a machine and its associat ed details

6
lib/threefold/tfgrid3deployer/webnames.v
View File

@@ -5,8 +5,10 @@ import json
@[params]
pub struct WebNameRequirements {
pub mut:
name string @[required]
node_id ?u32
name string @[required]
node_id ?u32
use_wireguard_network bool
use_hetzner_node bool
// must be in the format ip:port if tls_passthrough is set, otherwise the format should be http://ip[:port]
backend string @[required]
tls_passthrough bool

2
lib/threefold/zerohub/zerohub.v
View File

@@ -2,7 +2,7 @@ module zerohub
import net.http
// import freeflowuniverse.herolib.clients.httpconnection
// import freeflowuniverse.herolib.core.httpconnection
// TODO: curl -H "Authorization: bearer 6Pz6giOpHSaA3KdYI6LLpGSLmDmzmRkVdwvc7S-E5PVB0-iRfgDKW9Rb_ZTlj-xEW4_uSCa5VsyoRsML7DunA1sia3Jpc3RvZi4zYm90IiwgMTY3OTIxNTc3MF0=" https://hub.grid.tf/api/flist/

3
lib/virt/docker/docker_container_create.v
View File

@@ -1,6 +1,7 @@
module docker
import freeflowuniverse.herolib.osal { exec }
import freeflowuniverse.herolib.virt.utils
pub fn (mut e DockerEngine) container_create(args DockerContainerCreateArgs) !&DockerContainer {
mut ports := ''
@@ -33,7 +34,7 @@ pub fn (mut e DockerEngine) container_create(args DockerContainerCreateArgs) !&D
privileged := if args.privileged { '--privileged' } else { '' }
// if forwarded ports passed in the args not containing mapping tp ssh (22) create one
if !contains_ssh_port(args.forwarded_ports) {
if !utils.contains_ssh_port(args.forwarded_ports) {
// find random free port in the node
mut port := e.get_free_port() or { panic('No free port.') }
ports += '-p ${port}:22/tcp'

47
lib/virt/docker/docker_engine.v
View File

@@ -4,6 +4,7 @@ import freeflowuniverse.herolib.osal { exec }
import freeflowuniverse.herolib.core.texttools
import freeflowuniverse.herolib.virt.utils
import freeflowuniverse.herolib.core
import time
// import freeflowuniverse.herolib.installers.swarm
@@ -54,6 +55,29 @@ pub fn (mut e DockerEngine) load() ! {
e.containers_load()!
}
// load all images, they can be consulted in e.images
// see obj: DockerImage as result in e.images
pub fn (mut e DockerEngine) images_load() ! {
e.images = []DockerImage{}
mut lines := osal.execute_silent("docker images --format '{{.ID}}||{{.Repository}}||{{.Tag}}||{{.Digest}}||{{.Size}}||{{.CreatedAt}}'")!
for line in lines.split_into_lines() {
fields := line.split('||').map(utils.clear_str)
if fields.len != 6 {
panic('docker image needs to output 6 parts.\n${fields}')
}
mut image := DockerImage{
engine: &e
}
image.id = fields[0]
image.repo = fields[1]
image.tag = fields[2]
image.digest = utils.parse_digest(fields[3]) or { '' }
image.size = utils.parse_size_mb(fields[4]) or { 0 }
image.created = utils.parse_time(fields[5]) or { time.now() }
e.images << image
}
}
// load all containers, they can be consulted in e.containers
// see obj: DockerContainer as result in e.containers
pub fn (mut e DockerEngine) containers_load() ! {
@@ -65,7 +89,7 @@ pub fn (mut e DockerEngine) containers_load() ! {
stdout: false
)!
lines := ljob.output
for line in lines {
for line in lines.split_into_lines() {
if line.trim_space() == '' {
continue
}
@@ -142,22 +166,23 @@ pub fn (err ContainerGetError) code() int {
// image_id string
pub fn (mut e DockerEngine) containers_get(args_ ContainerGetArgs) ![]&DockerContainer {
mut args := args_
args.name = texttools.name_fix(args.name)
e.containers_load()!
mut res := []&DockerContainer{}
for _, c in e.containers {
for i, c in e.containers {
container := c
if args.name.contains('*') || args.name.contains('?') || args.name.contains('[') {
if c.name.match_glob(args.name) {
res << &c
if container.name.match_glob(args.name) {
res << &e.containers[i]
continue
}
} else {
if c.name == args.name || c.id == args.id {
res << &c
if container.name == args.name || container.id == args.id {
res << &e.containers[i]
continue
}
}
if args.image_id.len > 0 && c.image.id == args.image_id {
res << &c
if args.image_id.len > 0 && container.image.id == args.image_id {
res << &e.containers[i]
}
}
if res.len == 0 {
@@ -172,8 +197,8 @@ pub fn (mut e DockerEngine) containers_get(args_ ContainerGetArgs) ![]&DockerCon
// get container from memory, can use match_glob see https://modules.vlang.io/index.html#string.match_glob
pub fn (mut e DockerEngine) container_get(args_ ContainerGetArgs) !&DockerContainer {
mut args := args_
args.name = texttools.name_fix(args.name)
mut res := e.containers_get(args)!
if res.len > 1 {
return ContainerGetError{
args: args
@@ -211,7 +236,7 @@ pub fn (mut e DockerEngine) containers_delete(args ContainerGetArgs) ! {
// import a container into an image, run docker container with it
// image_repo examples ['myimage', 'myimage:latest']
// if DockerContainerCreateArgs contains a name, container will be created and restarted
pub fn (mut e DockerEngine) container_import(path string, mut args DockerContainerCreateArgs) !&DockerContainer {
pub fn (mut e DockerEngine) container_import(path string, args DockerContainerCreateArgs) !&DockerContainer {
mut image := args.image_repo
if args.image_tag != '' {
image = image + ':${args.image_tag}'

2
lib/virt/docker/docker_image.v
View File

@@ -89,7 +89,7 @@ pub fn (err ImageGetError) code() int {
pub fn (mut e DockerEngine) image_get(args ImageGetArgs) !&DockerImage {
mut counter := 0
mut result_digest := ''
for i in e.images {
for mut i in e.images {
if args.digest == args.digest {
return &i
}

9
lib/virt/docker/docker_recipe_code.v
View File

@@ -19,13 +19,10 @@ pub mut:
pub fn (mut r DockerBuilderRecipe) add_codeget(args_ CodeGetArgs) ! {
mut args := args_
mut gs := gittools.get(coderoot: '${r.path()}/code')!
locator := gs.locator_new(args.url)!
mut gr := gs.repo_get(locator: locator, pull: args.pull, reset: args.reset)!
mut gr := gs.get_repo(url: args.url, pull: args.pull, reset: args.reset)!
if args.name == '' {
args.name = gr.addr.name
args.name = gr.name
}
if args.dest == '' {
@@ -43,7 +40,7 @@ pub fn (mut r DockerBuilderRecipe) add_codeget(args_ CodeGetArgs) ! {
return error("dest is to short (min 3): now '${args.dest}'")
}
commonpath := gr.path_relative()
commonpath := gr.path()
if commonpath.contains('..') {
panic('bug should not be')
}

30
lib/virt/docker/docker_registry.v
View File

@@ -1,8 +1,8 @@
module docker
import freeflowuniverse.herolib.crypt.openssl
import freeflowuniverse.herolib.clients.httpconnection
import freeflowuniverse.herolib.osal { exec }
import freeflowuniverse.herolib.core.httpconnection
import freeflowuniverse.herolib.osal
import os
import freeflowuniverse.herolib.ui.console
@@ -84,30 +84,22 @@ pub fn (mut e DockerEngine) registry_add(args DockerRegistryArgs) ! {
e.registries << registry
// delete all previous containers, uses wildcards see https://modules.vlang.io/index.html#string.match_glob
e.container_delete(name: 'docker_registry*')!
e.container_delete(name: 'docker_registry*') or {
if !(err as ContainerGetError).notfound {
return err
}
println('No containers to matching docker registry')
}
composer.start()!
exec(cmd: 'curl https://localhost:5000/v2/ -k', retry: 4) or {
return error('could not start docker registry, did not answer')
}
mut conn := httpconnection.new(
name: 'localdockerhub'
url: 'https://localhost:5000/v2/'
retry: 10
)!
// r := conn.get_json_dict(mut prefix: 'errors')!
// r := conn.get_json_dict(mut prefix: 'errors')!
r := conn.get(method: .get)!
console.print_debug('Sdsd')
console.print_debug(r)
if true {
panic('sdsd')
}
// now we need to check if we can connect
res := conn.get()!
println(res)
}

2
lib/virt/hetzner/hetzner_model.v
View File

@@ -1,7 +1,7 @@
module hetzner
import freeflowuniverse.herolib.data.paramsparser
import freeflowuniverse.herolib.clients.httpconnection
import freeflowuniverse.herolib.core.httpconnection
pub const version = '1.14.3'
const singleton = false