MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

Merge branch 'development_ourdb_new' into development

Browse Source 
* development_ourdb_new: (115 commits)
  webdav completion wip
  Remove path from fsentry metadata, make vfs and webdav work again with fixes
  feat: Implement database synchronization using binary encoding
  Add documentation and tests for model_property.v
  feat: Add diagrams and README for OurDB syncer
  circle core objects work again
  ...
  ...
  radix tree has now prefix
  names
  models
  ...
  ....
  ...
  ...
  vfs_basics working
  vfs working
  ...
  ...
  ...
  ...

# Conflicts:
#	.gitignore
#	lib/code/generator/installer_client/ask.v
#	lib/code/generator/installer_client/factory.v
#	lib/code/generator/installer_client/generate.v
#	lib/code/generator/installer_client/model.v
#	lib/code/generator/installer_client/readme.md
#	lib/code/generator/installer_client/scanner.v
#	lib/code/generator/installer_client/templates/atemplate.yaml
#	lib/code/generator/installer_client/templates/heroscript_client
#	lib/code/generator/installer_client/templates/heroscript_installer
#	lib/code/generator/installer_client/templates/objname_actions.vtemplate
#	lib/code/generator/installer_client/templates/objname_factory_.vtemplate
#	lib/code/generator/installer_client/templates/objname_model.vtemplate
#	lib/code/generator/installer_client/templates/readme.md
#	lib/code/generator/installer_client_OLD/ask.v
#	lib/code/generator/installer_client_OLD/do.v
#	lib/code/generator/installer_client_OLD/generate.v
#	lib/code/generator/installer_client_OLD/model.v
#	lib/code/generator/installer_client_OLD/readme.md
#	lib/code/generator/installer_client_OLD/scanner.v
#	lib/code/generator/installer_client_OLD/templates/atemplate.yaml
#	lib/code/generator/installer_client_OLD/templates/heroscript_client
#	lib/code/generator/installer_client_OLD/templates/heroscript_installer
#	lib/code/generator/installer_client_OLD/templates/objname_actions.vtemplate
#	lib/code/generator/installer_client_OLD/templates/objname_factory_.vtemplate
#	lib/code/generator/installer_client_OLD/templates/objname_model.vtemplate
#	lib/code/generator/installer_client_OLD/templates/readme.md
#	lib/core/generator/installer_client_OLD/ask.v
#	lib/core/generator/installer_client_OLD/factory.v
#	lib/core/generator/installer_client_OLD/generate.v
#	lib/core/generator/installer_client_OLD/model.v
#	lib/core/generator/installer_client_OLD/readme.md
#	lib/core/generator/installer_client_OLD/scanner.v
#	lib/core/generator/installer_client_OLD/templates/atemplate.yaml
#	lib/core/generator/installer_client_OLD/templates/heroscript_client
#	lib/core/generator/installer_client_OLD/templates/heroscript_installer
#	lib/core/generator/installer_client_OLD/templates/objname_actions.vtemplate
#	lib/core/generator/installer_client_OLD/templates/objname_factory_.vtemplate
#	lib/core/generator/installer_client_OLD/templates/objname_model.vtemplate
#	lib/core/generator/installer_client_OLD/templates/readme.md
#	lib/core/texttools/namefix.v
This commit is contained in:
despiegk
2025-03-24 05:29:46 +01:00
parent 2e5210a0b7 a974091442
commit 9b0da9f245
436 changed files with 23966 additions and 5190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
lib/core/generator/generic/factory.v → lib/core/generator/generic/generate.v
View File

0
lib/core/generator/generic/generator.v → lib/core/generator/generic/generate_installer_client.v
View File

5
lib/core/generator/generic/model.v
View File

@@ -20,6 +20,9 @@ pub mut:
path string
force bool
hasconfig bool = true
playonly bool
play_name string // e.g. docusaurus is what we look for
module_path string // e.g.freeflowuniverse.herolib.web.docusaurus
}
pub enum Cat {
@@ -80,5 +83,5 @@ fn args_get(path string) !GeneratorArgs {
}
}
return error("can't find hero_code.generate_client or hero_code.generate_installer in ${path}")
// return GeneratorArgs{}
}

30
lib/core/generator/installer_client_OLD/factory.v
View File

@@ -9,10 +9,18 @@ pub mut:
reset bool // regenerate all, dangerous !!!
interactive bool // if we want to ask
path string
playonly bool
model ?GenModel
cat ?Cat
}
pub struct PlayArgs {
pub mut:
name string
modulepath string
}
// the default to start with
//
// reset bool // regenerate all, dangerous !!!
@@ -20,7 +28,9 @@ pub mut:
// path string
// model ?GenModel
// cat ?Cat
pub fn do(args_ GenerateArgs) ! {
//
// will return the module path where we need to execute a play command as well as the name of
pub fn do(args_ GenerateArgs) ! PlayArgs{
mut args := args_
console.print_header('Generate code for path: ${args.path} (reset:${args.reset}, interactive:${args.interactive})')
@@ -51,9 +61,9 @@ pub fn do(args_ GenerateArgs) ! {
}
}
if model.cat == .unknown {
model.cat = args.cat or { return error('cat needs to be specified for generator.') }
}
// if model.cat == .unknown {
// model.cat = args.cat or { return error('cat needs to be specified for generator.') }
// }
if args.interactive {
ask(args.path)!
@@ -64,5 +74,15 @@ pub fn do(args_ GenerateArgs) ! {
console.print_debug(args)
generate(args)!
//only generate if playonly is false and there is a classname
if !args.playonly && model.classname.len>0{
generate(args)!
}
return PlayArgs{
name: model.play_name
modulepath: model.module_path
}
}

29
lib/core/generator/installer_client_OLD/model.v
View File

@@ -20,6 +20,8 @@ pub mut:
build bool = true
hasconfig bool = true
cat Cat // dont' set default
play_name string // e.g. docusaurus is what we look for
module_path string // e.g.freeflowuniverse.herolib.web.docusaurus
}
pub enum Cat {
@@ -37,7 +39,6 @@ pub fn gen_model_set(args GenerateArgs) ! {
.installer { $tmpl('templates/heroscript_installer') }
else { return error('Invalid category: ${model.cat}') }
}
pathlib.template_write(heroscript_templ, '${args.path}/.heroscript', true)!
}
@@ -108,8 +109,30 @@ pub fn gen_model_get(path string, create bool) !GenModel {
model.name = os.base(path).to_lower()
}
console.print_debug('Code generator get: ${model}')
model.play_name = model.name
pathsub:=path.replace('${os.home_dir()}/code/github/','')
model.module_path = pathsub.replace("/",".").replace(".lib.",".")
// !!hero_code.play
// name:'docusaurus'
mut play_actions := plbook.find(filter: 'hero_code.play')!
if play_actions.len>1{
return error("should have max 1 hero_code.play action in ${config_path.path}")
}
if play_actions.len==1{
mut p := play_actions[0].params
model.play_name = p.get_default('name',model.name)!
}
if model.module_path.contains("docusaurus"){
println(model)
println("4567ujhjk")
exit(0)
}
return model
// return GenModel{}
}

2
lib/core/generator/installer_client_OLD/readme.md
View File

@@ -1,6 +1,8 @@
# generation framework for clients & installers
```bash
#generate all play commands
hero generate -playonly
#will ask questions if .heroscript is not there yet
hero generate -p thepath_is_optional
# to generate without questions

4
lib/core/generator/installer_client_OLD/scanner.v
View File

@@ -10,6 +10,7 @@ pub mut:
reset bool // regenerate all, dangerous !!!
interactive bool // if we want to ask
path string
playonly bool
}
// scan over a set of directories call the play where
@@ -19,6 +20,7 @@ pub fn scan(args ScannerArgs) ! {
if args.path == '' {
scan(path: '${os.home_dir()}/code/github/freeflowuniverse/herolib/lib/installers')!
scan(path: '${os.home_dir()}/code/github/freeflowuniverse/herolib/lib/clients')!
scan(path: '${os.home_dir()}/code/github/freeflowuniverse/herolib/lib/web')!
return
}
@@ -36,7 +38,7 @@ pub fn scan(args ScannerArgs) ! {
pparent := p.parent()!
path_module := pparent.path
if os.exists('${path_module}/.heroscript') {
do(interactive: args.interactive, path: path_module, reset: args.reset)!
do(interactive: args.interactive, path: path_module, reset: args.reset, playonly:args.playonly)!
}
}
}

14
lib/core/herocmds/generator.v
View File

@@ -36,6 +36,13 @@ pub fn cmd_generator(mut cmdroot Command) {
description: 'will work non interactive if possible.'
})
cmd_run.add_flag(Flag{
flag: .bool
required: false
name: 'playonly'
description: 'generate the play script.'
})
cmd_run.add_flag(Flag{
flag: .bool
required: false
@@ -59,9 +66,14 @@ fn cmd_generator_execute(cmd Command) ! {
mut force := cmd.flags.get_bool('force') or { false }
mut reset := cmd.flags.get_bool('reset') or { false }
mut scan := cmd.flags.get_bool('scan') or { false }
mut playonly := cmd.flags.get_bool('playonly') or { false }
mut installer := cmd.flags.get_bool('installer') or { false }
mut path := cmd.flags.get_string('path') or { '' }
if playonly{
force=true
}
if path == '' {
path = os.getwd()
}
@@ -74,7 +86,7 @@ fn cmd_generator_execute(cmd Command) ! {
}
if scan {
generic.scan(path: path, reset: reset, force: force, cat: cat)!
generic.scan(path: path, reset: reset, force: force, cat: cat, playonly:playonly)!
} else {
generic.generate(path: path, reset: reset, force: force, cat: cat)!
}

60
lib/core/jobs/model/agent.v
View File

@@ -1,60 +0,0 @@
module model
import freeflowuniverse.herolib.data.ourtime
// Agent represents a service provider that can execute jobs
pub struct Agent {
pub mut:
pubkey string // pubkey using ed25519
address string // where we can find the agent
port int // default 9999
description string // optional
status AgentStatus
services []AgentService // these are the public services
signature string // signature as done by private key of $address+$port+$description+$status
}
// AgentStatus represents the current state of an agent
pub struct AgentStatus {
pub mut:
guid string // unique id for the job
timestamp_first ourtime.OurTime // when agent came online
timestamp_last ourtime.OurTime // last time agent let us know that he is working
status AgentState // current state of the agent
}
// AgentService represents a service provided by an agent
pub struct AgentService {
pub mut:
actor string // name of the actor providing the service
actions []AgentServiceAction // available actions for this service
description string // optional description
status AgentServiceState // current state of the service
}
// AgentServiceAction represents an action that can be performed by a service
pub struct AgentServiceAction {
pub mut:
action string // which action
description string // optional description
params map[string]string // e.g. name:'name of the vm' ...
params_example map[string]string // e.g. name:'myvm'
status AgentServiceState // current state of the action
public bool // if everyone can use then true, if restricted means only certain people can use
}
// AgentState represents the possible states of an agent
pub enum AgentState {
ok // agent is functioning normally
down // agent is not responding
error // agent encountered an error
halted // agent has been manually stopped
}
// AgentServiceState represents the possible states of an agent service or action
pub enum AgentServiceState {
ok // service/action is functioning normally
down // service/action is not available
error // service/action encountered an error
halted // service/action has been manually stopped
}

91
lib/core/jobs/model/agent_manager.v
View File

@@ -1,91 +0,0 @@
module model
import freeflowuniverse.herolib.core.redisclient
import freeflowuniverse.herolib.data.ourtime
import json
const agents_key = 'herorunner:agents' // Redis key for storing agents
// AgentManager handles all agent-related operations
pub struct AgentManager {
mut:
redis &redisclient.Redis
}
// new creates a new Agent instance
pub fn (mut m AgentManager) new() Agent {
return Agent{
pubkey: '' // Empty pubkey to be filled by caller
port: 9999 // Default port
status: AgentStatus{
guid: ''
timestamp_first: ourtime.now()
timestamp_last: ourtime.OurTime{}
status: .ok
}
services: []AgentService{}
}
}
// add adds a new agent to Redis
pub fn (mut m AgentManager) set(agent Agent) ! {
// Store agent in Redis hash where key is agent.pubkey and value is JSON of agent
agent_json := json.encode(agent)
m.redis.hset(agents_key, agent.pubkey, agent_json)!
}
// get retrieves an agent by its public key
pub fn (mut m AgentManager) get(pubkey string) !Agent {
agent_json := m.redis.hget(agents_key, pubkey)!
return json.decode(Agent, agent_json)
}
// list returns all agents
pub fn (mut m AgentManager) list() ![]Agent {
mut agents := []Agent{}
// Get all agents from Redis hash
agents_map := m.redis.hgetall(agents_key)!
// Convert each JSON value to Agent struct
for _, agent_json in agents_map {
agent := json.decode(Agent, agent_json)!
agents << agent
}
return agents
}
// delete removes an agent by its public key
pub fn (mut m AgentManager) delete(pubkey string) ! {
m.redis.hdel(agents_key, pubkey)!
}
// update_status updates just the status of an agent
pub fn (mut m AgentManager) update_status(pubkey string, status AgentState) ! {
mut agent := m.get(pubkey)!
agent.status.status = status
m.set(agent)!
}
// get_by_service returns all agents that provide a specific service
pub fn (mut m AgentManager) get_by_service(actor string, action string) ![]Agent {
mut matching_agents := []Agent{}
agents := m.list()!
for agent in agents {
for service in agent.services {
if service.actor != actor {
continue
}
for act in service.actions {
if act.action == action {
matching_agents << agent
break
}
}
}
}
return matching_agents
}

74
lib/core/jobs/model/agent_manager_test.v
View File

@@ -1,74 +0,0 @@
module model
import freeflowuniverse.herolib.core.redisclient
import freeflowuniverse.herolib.data.ourtime
fn test_agents_model() {
mut runner := new()!
// Create a new agent using the manager
mut agent := runner.agents.new()
agent.pubkey = 'test-agent-1'
agent.address = '127.0.0.1'
agent.description = 'Test Agent'
// Create a service action
mut action := AgentServiceAction{
action: 'start'
description: 'Start a VM'
params: {
'name': 'string'
}
params_example: {
'name': 'myvm'
}
status: .ok
public: true
}
// Create a service
mut service := AgentService{
actor: 'vm_manager'
actions: [action]
description: 'VM Management Service'
status: .ok
}
agent.services = [service]
// Add the agent
runner.agents.set(agent)!
// Get the agent and verify fields
retrieved_agent := runner.agents.get(agent.pubkey)!
assert retrieved_agent.pubkey == agent.pubkey
assert retrieved_agent.address == agent.address
assert retrieved_agent.description == agent.description
assert retrieved_agent.services.len == 1
assert retrieved_agent.services[0].actor == 'vm_manager'
assert retrieved_agent.status.status == .ok
// Update agent status
runner.agents.update_status(agent.pubkey, .down)!
updated_agent := runner.agents.get(agent.pubkey)!
assert updated_agent.status.status == .down
// Test get_by_service
agents := runner.agents.get_by_service('vm_manager', 'start')!
assert agents.len > 0
assert agents[0].pubkey == agent.pubkey
// List all agents
all_agents := runner.agents.list()!
assert all_agents.len > 0
assert all_agents[0].pubkey == agent.pubkey
// Delete the agent
runner.agents.delete(agent.pubkey)!
// Verify deletion
agents_after := runner.agents.list()!
for a in agents_after {
assert a.pubkey != agent.pubkey
}
}

37
lib/core/jobs/model/factory.v
View File

@@ -1,37 +0,0 @@
module model
import freeflowuniverse.herolib.core.redisclient
// HeroRunner is the main factory for managing jobs, agents, services and groups
pub struct HeroRunner {
mut:
redis &redisclient.Redis
pub mut:
jobs &JobManager
agents &AgentManager
services &ServiceManager
groups &GroupManager
}
// new creates a new HeroRunner instance
pub fn new() !&HeroRunner {
mut redis := redisclient.core_get()!
mut hr := &HeroRunner{
redis: redis
jobs: &JobManager{
redis: redis
}
agents: &AgentManager{
redis: redis
}
services: &ServiceManager{
redis: redis
}
groups: &GroupManager{
redis: redis
}
}
return hr
}

10
lib/core/jobs/model/group.v
View File

@@ -1,10 +0,0 @@
module model
// Group represents a collection of members (users or other groups)
pub struct Group {
pub mut:
guid string // unique id
name string // name of the group
description string // optional description
members []string // can be other group or member which is defined by pubkey
}

99
lib/core/jobs/model/group_manager.v
View File

@@ -1,99 +0,0 @@
module model
import freeflowuniverse.herolib.core.redisclient
import json
const groups_key = 'herorunner:groups' // Redis key for storing groups
// GroupManager handles all group-related operations
pub struct GroupManager {
mut:
redis &redisclient.Redis
}
// new creates a new Group instance
pub fn (mut m GroupManager) new() Group {
return Group{
guid: '' // Empty GUID to be filled by caller
members: []string{}
}
}
// add adds a new group to Redis
pub fn (mut m GroupManager) set(group Group) ! {
// Store group in Redis hash where key is group.guid and value is JSON of group
group_json := json.encode(group)
m.redis.hset(groups_key, group.guid, group_json)!
}
// get retrieves a group by its GUID
pub fn (mut m GroupManager) get(guid string) !Group {
group_json := m.redis.hget(groups_key, guid)!
return json.decode(Group, group_json)
}
// list returns all groups
pub fn (mut m GroupManager) list() ![]Group {
mut groups := []Group{}
// Get all groups from Redis hash
groups_map := m.redis.hgetall(groups_key)!
// Convert each JSON value to Group struct
for _, group_json in groups_map {
group := json.decode(Group, group_json)!
groups << group
}
return groups
}
// delete removes a group by its GUID
pub fn (mut m GroupManager) delete(guid string) ! {
m.redis.hdel(groups_key, guid)!
}
// add_member adds a member (user pubkey or group GUID) to a group
pub fn (mut m GroupManager) add_member(guid string, member string) ! {
mut group := m.get(guid)!
if member !in group.members {
group.members << member
m.set(group)!
}
}
// remove_member removes a member from a group
pub fn (mut m GroupManager) remove_member(guid string, member string) ! {
mut group := m.get(guid)!
group.members = group.members.filter(it != member)
m.set(group)!
}
pub fn (mut m GroupManager) get_user_groups(user_pubkey string) ![]Group {
mut user_groups := []Group{}
mut checked_groups := map[string]bool{}
groups := m.list()!
// Check each group
for group in groups {
check_group_membership(group, user_pubkey, groups, mut checked_groups, mut user_groups)
}
return user_groups
}
// Recursive function to check group membership
fn check_group_membership(group Group, user string, groups []Group, mut checked map[string]bool, mut result []Group) {
if group.guid in checked {
return
}
checked[group.guid] = true
if user in group.members {
result << group
// Check parent groups
for parent_group in groups {
if group.guid in parent_group.members {
check_group_membership(parent_group, user, groups, mut checked, mut result)
}
}
}
}

67
lib/core/jobs/model/group_manager_test.v
View File

@@ -1,67 +0,0 @@
module model
import freeflowuniverse.herolib.core.redisclient
fn test_groups() {
mut runner := new()!
// Create a new group using the manager
mut group := runner.groups.new()
group.guid = 'admin-group'
group.name = 'Administrators'
group.description = 'Administrator group with full access'
// Add the group
runner.groups.set(group)!
// Create a subgroup
mut subgroup := runner.groups.new()
subgroup.guid = 'vm-admins'
subgroup.name = 'VM Administrators'
subgroup.description = 'VM management administrators'
runner.groups.set(subgroup)!
// Add subgroup to main group
runner.groups.add_member(group.guid, subgroup.guid)!
// Add a user to the subgroup
runner.groups.add_member(subgroup.guid, 'user-1-pubkey')!
// Get the groups and verify fields
retrieved_group := runner.groups.get(group.guid)!
assert retrieved_group.guid == group.guid
assert retrieved_group.name == group.name
assert retrieved_group.description == group.description
assert retrieved_group.members.len == 1
assert retrieved_group.members[0] == subgroup.guid
retrieved_subgroup := runner.groups.get(subgroup.guid)!
assert retrieved_subgroup.members.len == 1
assert retrieved_subgroup.members[0] == 'user-1-pubkey'
// Test recursive group membership
user_groups := runner.groups.get_user_groups('user-1-pubkey')!
assert user_groups.len == 1
assert user_groups[0].guid == subgroup.guid
// Remove member from subgroup
runner.groups.remove_member(subgroup.guid, 'user-1-pubkey')!
updated_subgroup := runner.groups.get(subgroup.guid)!
assert updated_subgroup.members.len == 0
// List all groups
groups := runner.groups.list()!
assert groups.len == 2
// Delete the groups
runner.groups.delete(subgroup.guid)!
runner.groups.delete(group.guid)!
// Verify deletion
groups_after := runner.groups.list()!
for g in groups_after {
assert g.guid != group.guid
assert g.guid != subgroup.guid
}
}

52
lib/core/jobs/model/job.v
View File

@@ -1,52 +0,0 @@
module model
import freeflowuniverse.herolib.data.ourtime
// Job represents a task to be executed by an agent
pub struct Job {
pub mut:
guid string // unique id for the job
agents []string // the pub key of the agent(s) which will execute the command, only 1 will execute
source string // pubkey from the agent who asked for the job
circle string = 'default' // our digital life is organized in circles
context string = 'default' // is the high level context in which actors will execute the work inside a circle
actor string // e.g. vm_manager
action string // e.g. start
params map[string]string // e.g. id:10
timeout_schedule u16 = 60 // timeout before its picked up
timeout u16 = 3600 // timeout in sec
log bool = true
ignore_error bool // means if error will just exit and not raise, there will be no error reporting
ignore_error_codes []int // of we want to ignore certain error codes
debug bool // if debug will get more context
retry int // default there is no debug
status JobStatus
dependencies []JobDependency // will not execute until other jobs are done
}
// JobStatus represents the current state of a job
pub struct JobStatus {
pub mut:
guid string // unique id for the job
created ourtime.OurTime // when we created the job
start ourtime.OurTime // when the job needs to start
end ourtime.OurTime // when the job ended, can be in error
status Status // current status of the job
}
// JobDependency represents a dependency on another job
pub struct JobDependency {
pub mut:
guid string // unique id for the job
agents []string // the pub key of the agent(s) which can execute the command
}
// Status represents the possible states of a job
pub enum Status {
created // initial state
scheduled // job has been scheduled
planned // arrived where actor will execute the job
running // job is currently running
error // job encountered an error
ok // job completed successfully
}

68
lib/core/jobs/model/job_manager.v
View File

@@ -1,68 +0,0 @@
module model
import freeflowuniverse.herolib.core.redisclient
import freeflowuniverse.herolib.data.ourtime
import json
const jobs_key = 'herorunner:jobs' // Redis key for storing jobs
// JobManager handles all job-related operations
pub struct JobManager {
mut:
redis &redisclient.Redis
}
// new creates a new Job instance
pub fn (mut m JobManager) new() Job {
return Job{
guid: '' // Empty GUID to be filled by caller
status: JobStatus{
guid: ''
created: ourtime.now()
start: ourtime.OurTime{}
end: ourtime.OurTime{}
status: .created
}
}
}
// add adds a new job to Redis
pub fn (mut m JobManager) set(job Job) ! {
// Store job in Redis hash where key is job.guid and value is JSON of job
job_json := json.encode(job)
m.redis.hset(jobs_key, job.guid, job_json)!
}
// get retrieves a job by its GUID
pub fn (mut m JobManager) get(guid string) !Job {
job_json := m.redis.hget(jobs_key, guid)!
return json.decode(Job, job_json)
}
// list returns all jobs
pub fn (mut m JobManager) list() ![]Job {
mut jobs := []Job{}
// Get all jobs from Redis hash
jobs_map := m.redis.hgetall(jobs_key)!
// Convert each JSON value to Job struct
for _, job_json in jobs_map {
job := json.decode(Job, job_json)!
jobs << job
}
return jobs
}
// delete removes a job by its GUID
pub fn (mut m JobManager) delete(guid string) ! {
m.redis.hdel(jobs_key, guid)!
}
// update_status updates just the status of a job
pub fn (mut m JobManager) update_status(guid string, status Status) ! {
mut job := m.get(guid)!
job.status.status = status
m.set(job)!
}

47
lib/core/jobs/model/job_manager_test.v
View File

@@ -1,47 +0,0 @@
module model
import freeflowuniverse.herolib.core.redisclient
import freeflowuniverse.herolib.data.ourtime
fn test_jobs() {
mut runner := new()!
// Create a new job using the manager
mut job := runner.jobs.new()
job.guid = 'test-job-1'
job.actor = 'vm_manager'
job.action = 'start'
job.params = {
'id': '10'
}
// Add the job
runner.jobs.set(job)!
// Get the job and verify fields
retrieved_job := runner.jobs.get(job.guid)!
assert retrieved_job.guid == job.guid
assert retrieved_job.actor == job.actor
assert retrieved_job.action == job.action
assert retrieved_job.params['id'] == job.params['id']
assert retrieved_job.status.status == .created
// Update job status
runner.jobs.update_status(job.guid, .running)!
updated_job := runner.jobs.get(job.guid)!
assert updated_job.status.status == .running
// List all jobs
jobs := runner.jobs.list()!
assert jobs.len > 0
assert jobs[0].guid == job.guid
// Delete the job
runner.jobs.delete(job.guid)!
// Verify deletion
jobs_after := runner.jobs.list()!
for j in jobs_after {
assert j.guid != job.guid
}
}

44
lib/core/jobs/model/service.v
View File

@@ -1,44 +0,0 @@
module model
// Service represents a service that can be provided by agents
pub struct Service {
pub mut:
actor string // name of the actor providing the service
actions []ServiceAction // available actions for this service
description string // optional description
status ServiceState // current state of the service
acl ?ACL // access control list for the service
}
// ServiceAction represents an action that can be performed by a service
pub struct ServiceAction {
pub mut:
action string // which action
description string // optional description
params map[string]string // e.g. name:'name of the vm' ...
params_example map[string]string // e.g. name:'myvm'
acl ?ACL // if not used then everyone can use
}
// ACL represents an access control list
pub struct ACL {
pub mut:
name string
ace []ACE
}
// ACE represents an access control entry
pub struct ACE {
pub mut:
groups []string // guid's of the groups who have access
users []string // in case groups are not used then is users
right string // e.g. read, write, admin, block
}
// ServiceState represents the possible states of a service
pub enum ServiceState {
ok // service is functioning normally
down // service is not available
error // service encountered an error
halted // service has been manually stopped
}

122
lib/core/jobs/model/service_manager.v
View File

@@ -1,122 +0,0 @@
module model
import freeflowuniverse.herolib.core.redisclient
import json
const services_key = 'herorunner:services' // Redis key for storing services
// ServiceManager handles all service-related operations
pub struct ServiceManager {
mut:
redis &redisclient.Redis
}
// new creates a new Service instance
pub fn (mut m ServiceManager) new() Service {
return Service{
actor: '' // Empty actor name to be filled by caller
actions: []ServiceAction{}
status: .ok
}
}
// add adds a new service to Redis
pub fn (mut m ServiceManager) set(service Service) ! {
// Store service in Redis hash where key is service.actor and value is JSON of service
service_json := json.encode(service)
m.redis.hset(services_key, service.actor, service_json)!
}
// get retrieves a service by its actor name
pub fn (mut m ServiceManager) get(actor string) !Service {
service_json := m.redis.hget(services_key, actor)!
return json.decode(Service, service_json)
}
// list returns all services
pub fn (mut m ServiceManager) list() ![]Service {
mut services := []Service{}
// Get all services from Redis hash
services_map := m.redis.hgetall(services_key)!
// Convert each JSON value to Service struct
for _, service_json in services_map {
service := json.decode(Service, service_json)!
services << service
}
return services
}
// delete removes a service by its actor name
pub fn (mut m ServiceManager) delete(actor string) ! {
m.redis.hdel(services_key, actor)!
}
// update_status updates just the status of a service
pub fn (mut m ServiceManager) update_status(actor string, status ServiceState) ! {
mut service := m.get(actor)!
service.status = status
m.set(service)!
}
// get_by_action returns all services that provide a specific action
pub fn (mut m ServiceManager) get_by_action(action string) ![]Service {
mut matching_services := []Service{}
services := m.list()!
for service in services {
for act in service.actions {
if act.action == action {
matching_services << service
break
}
}
}
return matching_services
}
// check_access verifies if a user has access to a service action
pub fn (mut m ServiceManager) check_access(actor string, action string, user_pubkey string, groups []string) !bool {
service := m.get(actor)!
// Find the specific action
mut service_action := ServiceAction{}
mut found := false
for act in service.actions {
if act.action == action {
service_action = act
found = true
break
}
}
if !found {
return error('Action ${action} not found in service ${actor}')
}
// If no ACL is defined, access is granted
if service_action.acl == none {
return true
}
acl := service_action.acl or { return true }
// Check each ACE in the ACL
for ace in acl.ace {
// Check if user is directly listed
if user_pubkey in ace.users {
return ace.right != 'block'
}
// Check if any of user's groups are listed
for group in groups {
if group in ace.groups {
return ace.right != 'block'
}
}
}
return false
}

87
lib/core/jobs/model/service_manager_test.v
View File

@@ -1,87 +0,0 @@
module model
import freeflowuniverse.herolib.core.redisclient
fn test_services() {
mut runner := new()!
// Create a new service using the manager
mut service := runner.services.new()
service.actor = 'vm_manager'
service.description = 'VM Management Service'
// Create an ACL
mut ace := ACE{
groups: ['admin-group']
users: ['user-1-pubkey']
right: 'write'
}
mut acl := ACL{
name: 'vm-acl'
ace: [ace]
}
// Create a service action
mut action := ServiceAction{
action: 'start'
description: 'Start a VM'
params: {
'name': 'string'
}
params_example: {
'name': 'myvm'
}
acl: acl
}
service.actions = [action]
// Add the service
runner.services.set(service)!
// Get the service and verify fields
retrieved_service := runner.services.get(service.actor)!
assert retrieved_service.actor == service.actor
assert retrieved_service.description == service.description
assert retrieved_service.actions.len == 1
assert retrieved_service.actions[0].action == 'start'
assert retrieved_service.status == .ok
// Update service status
runner.services.update_status(service.actor, .down)!
updated_service := runner.services.get(service.actor)!
assert updated_service.status == .down
// Test get_by_action
services := runner.services.get_by_action('start')!
assert services.len > 0
assert services[0].actor == service.actor
// Test access control
has_access := runner.services.check_access(service.actor, 'start', 'user-1-pubkey',
[])!
assert has_access == true
has_group_access := runner.services.check_access(service.actor, 'start', 'user-2-pubkey',
['admin-group'])!
assert has_group_access == true
no_access := runner.services.check_access(service.actor, 'start', 'user-3-pubkey',
[])!
assert no_access == false
// List all services
all_services := runner.services.list()!
assert all_services.len > 0
assert all_services[0].actor == service.actor
// Delete the service
runner.services.delete(service.actor)!
// Verify deletion
services_after := runner.services.list()!
for s in services_after {
assert s.actor != service.actor
}
}

186
lib/core/jobs/model/specs.md
View File

@@ -1,186 +0,0 @@
create a job manager in
lib/core/jobs
## some definitions
- agent: is a self contained set of processes which can execute on actions or actions to be executed by others
- action: what needs to be executed
- circle: each action happens in a circle
- context: a context inside a circle is optional
- job, what gets executed by an agent, is one action, can depend on other actions
- herorunner: is the process which uses redis to manage all open jobs, checks for timeouts, does the forwards if needed (if remote agent need to schedule, ...)
## jobs
are executed by processes can be in different languages and they are identified by agent pub key (the one who executes)
as part of heroscript we know what to executed on which actor inside the agent, defined with method and its arguments
```v
//the description of what needs to be executed
pub struct Job {
pub mut:
guid string //unique id for the job
agents []string //the pub key of the agent(s) which will execute the command, only 1 will execute, the herorunner will try the different agents if needed till it has success
source string //pubkey from the agent who asked for the job
circle string = "default" //our digital life is organized in circles
context string = "default" //is the high level context in which actors will execute the work inside a circle
actor string //e.g. vm_manager
action string //e.g. start
params map[string]string //e.g. id:10
timeout_schedule u16 = 60 //timeout before its picked up
timeout u16 = 3600 // timeout in sec
log bool = true
ignore_error bool // means if error will just exit and not raise, there will be no error reporting
ignore_error_codes []int // of we want to ignore certain error codes
debug bool // if debug will get more context
retry int // default there is no debug
status JobStatus
dependencies []JobDependency //will not execute untill other jobs are done
}
pub struct JobStatus {
pub mut:
guid string //unique id for the job
created u32 //epoch when we created the job
start u32 //epoch when the job needs to start
end u32 //epoch when the job ended, can be in error
status //ENUM: create scheduled, planned (means arrived where actor will execute the job), running, error, ok
}
pub struct JobDependency {
pub mut:
guid string //unique id for the job
agents []string //the pub key of the agent(s) which can execute the command
}
```
the Job object is stored in redis in hset herorunner:jobs where key is the job guid and the val is the json of Job
## Agent Registration Services
Each agent (the one who hosts the different actors which execute the methods with params) register themselves to all participants.
the structs below are available to everyone and are public
```v
pub struct Agent {
pub mut:
pubkey string //pubkey using ed25519
address string //where we can gind the agent
port int //default 9999
description string //optional
status AgentStatus
services []AgentService //these are the public services
signature string //signature as done by private key of $address+$port+$description+$status (this allows everyone to verify that the data is ok)
}
pub struct AgentStatus {
pub mut:
guid string //unique id for the job
timestamp_first u32 //when agent came online
timestamp_last u32 //last time agent let us know that he is working
status //ENUM: ok, down, error, halted
}
pub struct AgentService {
pub mut:
actor string
actions []AgentServiceAction
description string
status //ENUM: ok, down, error, halted
}
pub struct AgentServiceAction {
pub mut:
action string //which action
description string //optional descroption
params map[string]string //e.g. name:'name of the vm' ...
params_example map[string]string // e.g. name:'myvm'
status //ENUM: ok, down, error, halted
public bool //if everyone can use then true, if restricted means only certain people can use
}
```
the Agent object is stored in redis in hset herorunner:agents where key is the agent pubkey and the val is the json of Agent
### Services Info
The agent and its actors register their capability to the herorunner
We have a mechanism to be specific on who can execute which, this is sort of ACL system, for now its quite rough
```v
pub struct Group {
pub mut:
guid string //unique id
name string
description string
members []string //can be other group or member which is defined by pubkey
}
```
this info is stored in in redis on herorunner:groups
```v
pub struct Service {
pub mut:
actor string
actions []AgentServiceAction
description string
status //ENUM: ok, down, error, halted
acl ?ACL
}
pub struct ServiceAction {
pub mut:
action string //which action
description string //optional descroption
params map[string]string //e.g. name:'name of the vm' ...
params_example map[string]string // e.g. name:'myvm'
acl ?ACL //if not used then everyone can use
}
pub struct ACL {
pub mut:
name string
ace []ACE
}
pub struct ACE {
pub mut:
groups []string //guid's of the groups who have access
users []string //in case groups are not used then is users
right string e.g. read, write, admin, block
}
```
The info for the herorunner to function is in redis on herorunner:services

0
lib/core/jobs/openrpc/.gitignore → lib/core/openrpc_remove/.gitignore vendored
View File

0
lib/core/jobs/openrpc/examples/job_client.vsh → lib/core/openrpc_remove/examples/job_client.vsh
View File

0
lib/core/jobs/openrpc/examples/server.vsh → lib/core/openrpc_remove/examples/server.vsh
View File

0
lib/core/jobs/openrpc/factory.v → lib/core/openrpc_remove/factory.v
View File

0
lib/core/jobs/openrpc/handler.v → lib/core/openrpc_remove/handler.v
View File

0
lib/core/jobs/openrpc/handler_agent_manager.v → lib/core/openrpc_remove/handler_agent_manager.v
View File

0
lib/core/jobs/openrpc/handler_group_manager.v → lib/core/openrpc_remove/handler_group_manager.v
View File

0
lib/core/jobs/openrpc/handler_job_manager.v → lib/core/openrpc_remove/handler_job_manager.v
View File

0
lib/core/jobs/openrpc/handler_service_manager.v → lib/core/openrpc_remove/handler_service_manager.v
View File

0
lib/core/jobs/openrpc/model.v → lib/core/openrpc_remove/model.v
View File

0
lib/core/jobs/openrpc/specs/agent_manager_openrpc.json → lib/core/openrpc_remove/specs/agent_manager_openrpc.json
View File

4
lib/core/jobs/openrpc/specs/group_manager_openrpc.json → lib/core/openrpc_remove/specs/group_manager_openrpc.json
View File

@@ -1,9 +1,9 @@
{
"openrpc": "1.2.6",
"info": {
"title": "Group Manager API",
"title": "Group DBSession API",
"version": "1.0.0",
"description": "An OpenRPC specification for Group Manager methods"
"description": "An OpenRPC specification for Group DBSession methods"
},
"servers": [
{

0
lib/core/jobs/openrpc/specs/job_manager_openrpc.json → lib/core/openrpc_remove/specs/job_manager_openrpc.json
View File

0
lib/core/jobs/openrpc/specs/service_manager_openrpc.json → lib/core/openrpc_remove/specs/service_manager_openrpc.json
View File

0
lib/core/jobs/openrpc/ws_server.v → lib/core/openrpc_remove/ws_server.v
View File

7
lib/core/pathlib/factory.v
View File

@@ -50,6 +50,9 @@ pub fn get_dir(args_ GetArgs) !Path {
mut p2 := get_no_check(args.path)
if args.check {
p2.check()
if args.delete {
p2.delete()!
}
p2.absolute()
if p2.exist == .no {
if args.create {
@@ -64,9 +67,7 @@ pub fn get_dir(args_ GetArgs) !Path {
if args.empty {
p2.empty()!
}
if args.delete {
p2.delete()!
}
}
return p2
}

1
lib/core/pathlib/path_copy.v
View File

@@ -61,4 +61,5 @@ pub fn (mut path Path) copy(args_ CopyArgs) ! {
dest.check()
}
}

15
lib/core/pathlib/path_list.v
View File

@@ -126,10 +126,17 @@ fn (mut path Path) list_internal(args ListArgsInternal) ![]Path {
}
}
mut addthefile := true
for r in args.regex {
if !(r.matches_string(item)) {
addthefile = false
mut addthefile := false
// If no regex patterns provided, include all files
if args.regex.len == 0 {
addthefile = true
} else {
// Include file if ANY regex pattern matches (OR operation)
for r in args.regex {
if r.matches_string(item) {
addthefile = true
break
}
}
}
if addthefile && !args.dirs_only {

47
lib/core/pathlib/readme.md
View File

@@ -43,7 +43,52 @@ if path.is_dir() { /* is directory */ }
if path.is_link() { /* is symlink */ }
```
## 3. Common File Operations
## 3. File Listing and Filtering
```v
// List all files in a directory (recursive by default)
mut dir := pathlib.get('/some/dir')
mut pathlist := dir.list()!
// List only files matching specific extensions using regex
mut pathlist_images := dir.list(
regex: [r'.*\.png$', r'.*\.jpg$', r'.*\.svg$', r'.*\.jpeg$'],
recursive: true
)!
// List only directories
mut pathlist_dirs := dir.list(
dirs_only: true,
recursive: true
)!
// List only files
mut pathlist_files := dir.list(
files_only: true,
recursive: false // only in current directory
)!
// Include symlinks in the results
mut pathlist_with_links := dir.list(
include_links: true
)!
// Don't ignore hidden files (those starting with . or _)
mut pathlist_all := dir.list(
ignoredefault: false
)!
// Access the resulting paths
for path in pathlist.paths {
println(path.path)
}
// Perform operations on all paths in the list
pathlist.copy('/destination/dir')!
pathlist.delete()!
```
## 4. Common File Operations
```v
// Empty a directory

47
lib/core/texttools/namefix.v
View File

@@ -84,11 +84,58 @@ pub fn name_fix_no_underscore(name string) string {
return x
}
<<<<<<< HEAD
// remove underscores and extension
pub fn name_fix_no_underscore_no_ext(name_ string) string {
return name_fix_keepext(name_).all_before_last('.').replace('_', '')
=======
pub fn name_fix_snake_to_pascal(name string) string {
x := name.replace('_', ' ')
p := x.title().replace(' ', '')
return p
}
pub fn name_fix_dot_notation_to_pascal(name string) string {
x := name.replace('.', ' ')
p := x.title().replace(' ', '')
return p
}
pub fn name_fix_pascal(name string) string {
name_ := name_fix_snake_to_pascal(name)
return name_fix_dot_notation_to_pascal(name_)
}
pub fn name_fix_pascal_to_snake(name string) string {
mut fixed := ''
for i, c in name {
if c.is_capital() && i != 0 {
fixed += '_'
}
fixed += c.ascii_str()
}
return fixed.to_lower()
}
pub fn name_fix_dot_notation_to_snake_case(name string) string {
return name.replace('.', '_')
}
// normalize a file path while preserving path structure
pub fn path_fix(path_ string) string {
if path_.len == 0 {
return ''
}
return "${path_.trim('/')}"
>>>>>>> development_ourdb_new
}
// normalize a file path while preserving path structure
pub fn path_fix_absolute(path string) string {
return "/${path_fix(path)}"
}
// remove underscores and extension
pub fn name_fix_no_ext(name_ string) string {
return name_fix_keepext(name_).all_before_last('.').trim_right('_')

30
lib/core/texttools/namefix_test.v
View File

@@ -6,3 +6,33 @@ fn test_main() {
assert name_fix_keepext('\$sds_?_!"`{_ 4F') == 'sds_4f'
assert name_fix_keepext('\$sds_?_!"`{_ 4F.jpg') == 'sds_4f.jpg'
}
fn test_path_fix() {
// Test empty path
assert path_fix('') == ''
// Test absolute paths
assert path_fix('/home/user') == '/home/user'
assert path_fix('/home/USER') == '/home/user'
assert path_fix('/home/user/Documents') == '/home/user/documents'
// Test relative paths
assert path_fix('home/user') == 'home/user'
assert path_fix('./home/user') == './home/user'
assert path_fix('../home/user') == '../home/user'
// Test paths with special characters
assert path_fix('/home/user/My Documents') == '/home/user/my_documents'
assert path_fix('/home/user/file-name.txt') == '/home/user/file_name.txt'
assert path_fix('/home/user/file name with spaces.txt') == '/home/user/file_name_with_spaces.txt'
// Test paths with multiple special characters
assert path_fix('/home/user/!@#$%^&*()_+.txt') == '/home/user/'
// Test paths with multiple components and extensions
assert path_fix('/home/user/Documents/report.pdf') == '/home/user/documents/report.pdf'
assert path_fix('/home/user/Documents/report.PDF') == '/home/user/documents/report.pdf'
// Test paths with multiple slashes
assert path_fix('/home//user///documents') == '/home/user/documents'
}