herocode/herolib_python
12
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
despiegk 2025-08-05 15:48:18 +02:00
parent 7fabb4163a
commit 824c71ef98
20 changed files with 1538 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
lib/core/logger/__init__.py Normal file
View File

BIN
lib/core/logger/__pycache__/__init__.cpython-313.pyc Normal file
View File

Binary file not shown.

BIN
lib/core/logger/__pycache__/factory.cpython-313.pyc Normal file
View File

Binary file not shown.

BIN
lib/core/logger/__pycache__/model.cpython-313.pyc Normal file
View File

Binary file not shown.

9
lib/core/logger/factory.py Normal file
View File

@ -0,0 +1,9 @@
from lib.core.pathlib.pathlib import get_dir
from lib.core.logger.model import Logger
def new(path: str) -> Logger:
p = get_dir(path=path, create=True)
return Logger(
path=p,
lastlog_time=0
)

822
lib/core/logger/instructions.md Normal file
View File

@ -0,0 +1,822 @@
<file_map>
/Users/despiegk/code/github/freeflowuniverse/herolib
├── aiprompts
│ └── herolib_core
│ ├── core_ourtime.md
│ ├── core_paths.md
│ └── core_text.md
└── lib
└── core
└── logger
├── factory.v
├── log_test.v
├── log.v
├── model.v
├── readme.md
└── search.v
</file_map>
<file_contents>
File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/factory.v
```v
module logger
import freeflowuniverse.herolib.core.pathlib
pub fn new(path string) !Logger {
mut p := pathlib.get_dir(path: path, create: true)!
return Logger{
path: p
lastlog_time: 0
}
}
```
File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/log_test.v
```v
module logger
import os
import freeflowuniverse.herolib.data.ourtime
import freeflowuniverse.herolib.core.pathlib
fn testsuite_begin() {
if os.exists('/tmp/testlogs') {
os.rmdir_all('/tmp/testlogs')!
}
}
fn test_logger() {
mut logger := new('/tmp/testlogs')!
// Test stdout logging
logger.log(LogItemArgs{
cat: 'test-app'
log: 'This is a test message\nWith a second line\nAnd a third line'
logtype: .stdout
timestamp: ourtime.new('2022-12-05 20:14:35')!
})!
// Test error logging
logger.log(LogItemArgs{
cat: 'error-test'
log: 'This is an error\nWith details'
logtype: .error
timestamp: ourtime.new('2022-12-05 20:14:35')!
})!
logger.log(LogItemArgs{
cat: 'test-app'
log: 'This is a test message\nWith a second line\nAnd a third line'
logtype: .stdout
timestamp: ourtime.new('2022-12-05 20:14:36')!
})!
logger.log(LogItemArgs{
cat: 'error-test'
log: '
This is an error
With details
'
logtype: .error
timestamp: ourtime.new('2022-12-05 20:14:36')!
})!
logger.log(LogItemArgs{
cat: 'error-test'
log: '
aaa
bbb
'
logtype: .error
timestamp: ourtime.new('2022-12-05 22:14:36')!
})!
logger.log(LogItemArgs{
cat: 'error-test'
log: '
aaa2
bbb2
'
logtype: .error
timestamp: ourtime.new('2022-12-05 22:14:36')!
})!
// Verify log directory exists
assert os.exists('/tmp/testlogs'), 'Log directory should exist'
// Get log file
files := os.ls('/tmp/testlogs')!
assert files.len == 2
mut file := pathlib.get_file(
path: '/tmp/testlogs/${files[0]}'
create: false
)!
content := file.read()!.trim_space()
items_stdout := logger.search(
timestamp_from: ourtime.new('2022-11-1 20:14:35')!
timestamp_to: ourtime.new('2025-11-1 20:14:35')!
logtype: .stdout
)!
assert items_stdout.len == 2
items_error := logger.search(
timestamp_from: ourtime.new('2022-11-1 20:14:35')!
timestamp_to: ourtime.new('2025-11-1 20:14:35')!
logtype: .error
)!
assert items_error.len == 4
}
fn testsuite_end() {
// if os.exists('/tmp/testlogs') {
// os.rmdir_all('/tmp/testlogs')!
// }
}
```
File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/log.v
```v
module logger
import os
import freeflowuniverse.herolib.core.texttools
import freeflowuniverse.herolib.data.ourtime
@[params]
pub struct LogItemArgs {
pub mut:
timestamp ?ourtime.OurTime
cat string
log string
logtype LogType
}
pub fn (mut l Logger) log(args_ LogItemArgs) ! {
mut args := args_
t := args.timestamp or {
t2 := ourtime.now()
t2
}
// Format category (max 10 chars, ascii only)
args.cat = texttools.name_fix(args.cat)
if args.cat.len > 10 {
return error('category cannot be longer than 10 chars')
}
args.cat = texttools.expand(args.cat, 10, ' ')
args.log = texttools.dedent(args.log).trim_space()
mut logfile_path := '${l.path.path}/${t.dayhour()}.log'
// Create log file if it doesn't exist
if !os.exists(logfile_path) {
os.write_file(logfile_path, '')!
l.lastlog_time = 0 // make sure we put time again
}
mut f := os.open_append(logfile_path)!
mut content := ''
// Add timestamp if we're in a new second
if t.unix() > l.lastlog_time {
content += '\n${t.time().format_ss()}\n'
l.lastlog_time = t.unix()
}
// Format log lines
error_prefix := if args.logtype == .error { 'E' } else { ' ' }
lines := args.log.split('\n')
for i, line in lines {
if i == 0 {
content += '${error_prefix} ${args.cat} - ${line}\n'
} else {
content += '${error_prefix} ${line}\n'
}
}
f.writeln(content.trim_space_right())!
f.close()
}
```
File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/model.v
```v
module logger
import freeflowuniverse.herolib.data.ourtime
import freeflowuniverse.herolib.core.pathlib
@[heap]
pub struct Logger {
pub mut:
path pathlib.Path
lastlog_time i64 // to see in log format, every second we put a time down, we need to know if we are in a new second (logs can come in much faster)
}
pub struct LogItem {
pub mut:
timestamp ourtime.OurTime
cat string
log string
logtype LogType
}
pub enum LogType {
stdout
error
}
```
File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/readme.md
```md
# Logger Module
A simple logging system that provides structured logging with search capabilities.
Logs are stored in hourly files with a consistent format that makes them both human-readable and machine-parseable.
## Features
- Structured logging with categories and error types
- Automatic timestamp management
- Multi-line message support
- Search functionality with filtering options
- Human-readable log format
## Usage
```v
import freeflowuniverse.herolib.core.logger
import freeflowuniverse.herolib.data.ourtime
// Create a new logger
mut l := logger.new(path: '/var/logs')!
// Log a message
l.log(
cat: 'system',
log: 'System started successfully',
logtype: .stdout
)!
// Log an error
l.log(
cat: 'system',
log: 'Failed to connect\nRetrying in 5 seconds...',
logtype: .error
)!
// Search logs
results := l.search(
timestamp_from: ourtime.now().warp("-24h"), // Last 24 hours
cat: 'system', // Filter by category
log: 'failed', // Search in message content
logtype: .error, // Only error messages
maxitems: 100 // Limit results
)!
```
## Log Format
Each log file is named using the format `YYYY-MM-DD-HH.log` and contains entries in the following format:
```
21:23:42
system - This is a normal log message
system - This is a multi-line message
second line with proper indentation
third line maintaining alignment
E error_cat - This is an error message
E second line of error
E third line of error
```
### Format Rules
- Time stamps (HH:MM:SS) are written once per second when the log time changes
- Categories are:
- Limited to 10 characters maximum
- Padded with spaces to exactly 10 characters
- Any `-` in category names are converted to `_`
- Each line starts with either:
- ` ` (space) for normal logs (LogType.stdout)
- `E` for error logs (LogType.error)
- Multi-line messages maintain consistent indentation (14 spaces after the prefix)
```
File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/search.v
```v
module logger
import os
import freeflowuniverse.herolib.core.texttools
import freeflowuniverse.herolib.data.ourtime
@[params]
pub struct SearchArgs {
pub mut:
timestamp_from ?ourtime.OurTime
timestamp_to ?ourtime.OurTime
cat string // can be empty
log string // any content in here will be looked for
logtype LogType
maxitems int = 10000
}
pub fn (mut l Logger) search(args_ SearchArgs) ![]LogItem {
mut args := args_
// Format category (max 10 chars, ascii only)
args.cat = texttools.name_fix(args.cat)
if args.cat.len > 10 {
return error('category cannot be longer than 10 chars')
}
mut timestamp_from := args.timestamp_from or { ourtime.OurTime{} }
mut timestamp_to := args.timestamp_to or { ourtime.OurTime{} }
// Get time range
from_time := timestamp_from.unix()
to_time := timestamp_to.unix()
if from_time > to_time {
return error('from_time cannot be after to_time: ${from_time} < ${to_time}')
}
mut result := []LogItem{}
// Find log files in time range
mut files := os.ls(l.path.path)!
files.sort()
for file in files {
if !file.ends_with('.log') {
continue
}
// Parse dayhour from filename
dayhour := file[..file.len - 4] // remove .log
file_time := ourtime.new(dayhour)!
mut current_time := ourtime.OurTime{}
mut current_item := LogItem{}
mut collecting := false
// Skip if file is outside time range
if file_time.unix() < from_time || file_time.unix() > to_time {
continue
}
// Read and parse log file
content := os.read_file('${l.path.path}/${file}')!
lines := content.split('\n')
for line in lines {
if result.len >= args.maxitems {
return result
}
line_trim := line.trim_space()
if line_trim == '' {
continue
}
// Check if this is a timestamp line
if !(line.starts_with(' ') || line.starts_with('E')) {
current_time = ourtime.new(line_trim)!
if collecting {
process(mut result, current_item, current_time, args, from_time, to_time)!
}
collecting = false
continue
}
if collecting && line.len > 14 && line[13] == `-` {
process(mut result, current_item, current_time, args, from_time, to_time)!
collecting = false
}
// Parse log line
is_error := line.starts_with('E')
if !collecting {
// Start new item
current_item = LogItem{
timestamp: current_time
cat: line[2..12].trim_space()
log: line[15..].trim_space()
logtype: if is_error { .error } else { .stdout }
}
// println('new current item: ${current_item}')
collecting = true
} else {
// Continuation line
if line_trim.len < 16 {
current_item.log += '\n'
} else {
current_item.log += '\n' + line[15..]
}
}
}
// Add last item if collecting
if collecting {
process(mut result, current_item, current_time, args, from_time, to_time)!
}
}
return result
}
fn process(mut result []LogItem, current_item LogItem, current_time ourtime.OurTime, args SearchArgs, from_time i64, to_time i64) ! {
// Add previous item if it matches filters
log_epoch := current_item.timestamp.unix()
if log_epoch < from_time || log_epoch > to_time {
return
}
if (args.cat == '' || current_item.cat.trim_space() == args.cat)
&& (args.log == '' || current_item.log.contains(args.log))
&& args.logtype == current_item.logtype {
result << current_item
}
}
```
File: /Users/despiegk/code/github/freeflowuniverse/herolib/aiprompts/herolib_core/core_ourtime.md
```md
# OurTime Module
The `OurTime` module in V provides flexible time handling, supporting relative and absolute time formats, Unix timestamps, and formatting utilities.
## Key Features
- Create time objects from strings or current time
- Relative time expressions (e.g., `+1h`, `-2d`)
- Absolute time formats (e.g., `YYYY-MM-DD HH:mm:ss`)
- Unix timestamp conversion
- Time formatting and warping
## Basic Usage
```v
import freeflowuniverse.herolib.data.ourtime
// Current time
mut t := ourtime.now()
// From string
t2 := ourtime.new('2022-12-05 20:14:35')!
// Get formatted string
println(t2.str()) // e.g., 2022-12-05 20:14
// Get Unix timestamp
println(t2.unix()) // e.g., 1670271275
```
## Time Formats
### Relative Time
Use `s` (seconds), `h` (hours), `d` (days), `w` (weeks), `M` (months), `Q` (quarters), `Y` (years).
```v
// Create with relative time
mut t := ourtime.new('+1w +2d -4h')!
// Warp existing time
mut t2 := ourtime.now()
t2.warp('+1h')!
```
### Absolute Time
Supports `YYYY-MM-DD HH:mm:ss`, `YYYY-MM-DD HH:mm`, `YYYY-MM-DD HH`, `YYYY-MM-DD`, `DD-MM-YYYY`.
```v
t1 := ourtime.new('2022-12-05 20:14:35')!
t2 := ourtime.new('2022-12-05')! // Time defaults to 00:00:00
```
## Methods Overview
### Creation
```v
now_time := ourtime.now()
from_string := ourtime.new('2023-01-15')!
from_epoch := ourtime.new_from_epoch(1673788800)
```
### Formatting
```v
mut t := ourtime.now()
println(t.str()) // YYYY-MM-DD HH:mm
println(t.day()) // YYYY-MM-DD
println(t.key()) // YYYY_MM_DD_HH_mm_ss
println(t.md()) // Markdown format
```
### Operations
```v
mut t := ourtime.now()
t.warp('+1h')! // Move 1 hour forward
unix_ts := t.unix()
is_empty := t.empty()
```
## Error Handling
Time parsing methods return a `Result` type and should be handled with `!` or `or` blocks.
```v
t_valid := ourtime.new('2023-01-01')!
t_invalid := ourtime.new('bad-date') or {
println('Error: ${err}')
ourtime.now() // Fallback
}
```
File: /Users/despiegk/code/github/freeflowuniverse/herolib/aiprompts/herolib_core/core_paths.md
```md
# Pathlib Usage Guide
## Overview
The pathlib module provides a comprehensive interface for handling file system operations. Key features include:
- Robust path handling for files, directories, and symlinks
- Support for both absolute and relative paths
- Automatic home directory expansion (~)
- Recursive directory operations
- Path filtering and listing
- File and directory metadata access
## Basic Usage
### Importing pathlib
```v
import freeflowuniverse.herolib.core.pathlib
```
### Creating Path Objects
```v
// Create a Path object for a file
mut file_path := pathlib.get("path/to/file.txt")
// Create a Path object for a directory
mut dir_path := pathlib.get("path/to/directory")
```
### Basic Path Operations
```v
// Get absolute path
abs_path := file_path.absolute()
// Get real path (resolves symlinks)
real_path := file_path.realpath()
// Check if path exists
if file_path.exists() {
// Path exists
}
```
## Path Properties and Methods
### Path Types
```v
// Check if path is a file
if file_path.is_file() {
// Handle as file
}
// Check if path is a directory
if dir_path.is_dir() {
// Handle as directory
}
// Check if path is a symlink
if file_path.is_link() {
// Handle as symlink
}
```
### Path Normalization
```v
// Normalize path (remove extra slashes, resolve . and ..)
normalized_path := file_path.path_normalize()
// Get path directory
dir_path := file_path.path_dir()
// Get path name without extension
name_no_ext := file_path.name_no_ext()
```
## File and Directory Operations
### File Operations
```v
// Write to file
file_path.write("Content to write")!
// Read from file
content := file_path.read()!
// Delete file
file_path.delete()!
```
### Directory Operations
```v
// Create directory
mut dir := pathlib.get_dir(
path: "path/to/new/dir"
create: true
)!
// List directory contents
mut dir_list := dir.list()!
// Delete directory
dir.delete()!
```
### Symlink Operations
```v
// Create symlink
file_path.link("path/to/symlink", delete_exists: true)!
// Resolve symlink
real_path := file_path.realpath()
```
## Advanced Operations
### Path Copying
```v
// Copy file to destination
file_path.copy(dest: "path/to/destination")!
```
### Recursive Operations
```v
// List directory recursively
mut recursive_list := dir.list(recursive: true)!
// Delete directory recursively
dir.delete()!
```
### Path Filtering
```v
// List files matching pattern
mut filtered_list := dir.list(
regex: [r".*\.txt$"],
recursive: true
)!
```
## Best Practices
### Error Handling
```v
if file_path.exists() {
// Safe to operate
} else {
// Handle missing file
}
```
```
File: /Users/despiegk/code/github/freeflowuniverse/herolib/aiprompts/herolib_core/core_text.md
```md
# TextTools Module
The `texttools` module provides a comprehensive set of utilities for text manipulation and processing.
## Functions and Examples:
```v
import freeflowuniverse.herolib.core.texttools
assert hello_world == texttools.name_fix("Hello World!")
```
### Name/Path Processing
* `name_fix(name string) string`: Normalizes filenames and paths.
* `name_fix_keepspace(name string) !string`: Like name_fix but preserves spaces.
* `name_fix_no_ext(name_ string) string`: Removes file extension.
* `name_fix_snake_to_pascal(name string) string`: Converts snake_case to PascalCase.
```v
name := texttools.name_fix_snake_to_pascal("hello_world") // Result: "HelloWorld"
```
* `snake_case(name string) string`: Converts PascalCase to snake_case.
```v
name := texttools.snake_case("HelloWorld") // Result: "hello_world"
```
* `name_split(name string) !(string, string)`: Splits name into site and page components.
### Text Cleaning
* `name_clean(r string) string`: Normalizes names by removing special characters.
```v
name := texttools.name_clean("Hello@World!") // Result: "HelloWorld"
```
* `ascii_clean(r string) string`: Removes all non-ASCII characters.
* `remove_empty_lines(text string) string`: Removes empty lines from text.
```v
text := texttools.remove_empty_lines("line1\n\nline2\n\n\nline3") // Result: "line1\nline2\nline3"
```
* `remove_double_lines(text string) string`: Removes consecutive empty lines.
* `remove_empty_js_blocks(text string) string`: Removes empty code blocks (```...```).
### Command Line Parsing
* `cmd_line_args_parser(text string) ![]string`: Parses command line arguments with support for quotes and escaping.
```v
args := texttools.cmd_line_args_parser("'arg with spaces' --flag=value") // Result: ['arg with spaces', '--flag=value']
```
* `text_remove_quotes(text string) string`: Removes quoted sections from text.
* `check_exists_outside_quotes(text string, items []string) bool`: Checks if items exist in text outside of quotes.
### Text Expansion
* `expand(txt_ string, l int, expand_with string) string`: Expands text to a specified length with a given character.
### Indentation
* `indent(text string, prefix string) string`: Adds indentation prefix to each line.
```v
text := texttools.indent("line1\nline2", " ") // Result: " line1\n line2\n"
```
* `dedent(text string) string`: Removes common leading whitespace from every line.
```v
text := texttools.dedent(" line1\n line2") // Result: "line1\nline2"
```
### String Validation
* `is_int(text string) bool`: Checks if text contains only digits.
* `is_upper_text(text string) bool`: Checks if text contains only uppercase letters.
### Multiline Processing
* `multiline_to_single(text string) !string`: Converts multiline text to a single line with proper escaping.
### Text Splitting
* `split_smart(t string, delimiter_ string) []string`: Intelligent string splitting that respects quotes.
### Tokenization
* `tokenize(text_ string) TokenizerResult`: Tokenizes text into meaningful parts.
* `text_token_replace(text string, tofind string, replacewith string) !string`: Replaces tokens in text.
### Version Parsing
* `version(text_ string) int`: Converts version strings to comparable integers.
```v
ver := texttools.version("v0.4.36") // Result: 4036
ver = texttools.version("v1.4.36") // Result: 1004036
```
### Formatting
* `format_rfc1123(t time.Time) string`: Formats a time.Time object into RFC 1123 format.
### Array Operations
* `to_array(r string) []string`: Converts a comma or newline separated list to an array of strings.
```v
text := "item1,item2,item3"
array := texttools.to_array(text) // Result: ['item1', 'item2', 'item3']
```
* `to_array_int(r string) []int`: Converts a text list to an array of integers.
* `to_map(mapstring string, line string, delimiter_ string) map[string]string`: Intelligent mapping of a line to a map based on a template.
```v
r := texttools.to_map("name,-,-,-,-,pid,-,-,-,-,path",
"root 304 0.0 0.0 408185328 1360 ?? S 16Dec23 0:34.06 /usr/sbin/distnoted")
// Result: {'name': 'root', 'pid': '1360', 'path': '/usr/sbin/distnoted'}
```
```
</file_contents>
<user_instructions>
create a module in python in location lib/core/logger in herolib_python
which reimplements /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger
all features need to be reimplemented
write me an implementation plan for my coding agent
</user_instructions>

3
lib/core/logger/log.py Normal file
View File

@ -0,0 +1,3 @@
# This file is now empty as the log function has been moved to model.py
# It can be removed or kept as a placeholder if needed for future extensions.
# For now, we will keep it empty.

150
lib/core/logger/log_test.py Normal file
View File

@ -0,0 +1,150 @@
import unittest
import os
import shutil
from lib.core.logger.factory import new
from lib.core.logger.model import LogItemArgs, LogType, Logger # Import Logger class
from lib.data.ourtime.ourtime import new as ourtime_new, now as ourtime_now
from lib.core.pathlib.pathlib import get_file, ls, rmdir_all
class TestLogger(unittest.TestCase):
def setUp(self):
# Corresponds to testsuite_begin()
if os.path.exists('/tmp/testlogs'):
rmdir_all('/tmp/testlogs')
def tearDown(self):
# Corresponds to testsuite_end()
# if os.path.exists('/tmp/testlogs'):
# rmdir_all('/tmp/testlogs')
pass
def test_logger_functionality(self):
logger = new('/tmp/testlogs')
# Test stdout logging
logger.log(LogItemArgs(
cat='test-app',
log='This is a test message\nWith a second line\nAnd a third line',
logtype=LogType.STDOUT,
timestamp=ourtime_new('2022-12-05 20:14:35')
))
# Test error logging
logger.log(LogItemArgs(
cat='error-test',
log='This is an error\nWith details',
logtype=LogType.ERROR,
timestamp=ourtime_new('2022-12-05 20:14:35')
))
logger.log(LogItemArgs(
cat='test-app',
log='This is a test message\nWith a second line\nAnd a third line',
logtype=LogType.STDOUT,
timestamp=ourtime_new('2022-12-05 20:14:36')
))
logger.log(LogItemArgs(
cat='error-test',
log='''
This is an error
With details
''',
logtype=LogType.ERROR,
timestamp=ourtime_new('2022-12-05 20:14:36')
))
logger.log(LogItemArgs(
cat='error-test',
log='''
aaa
bbb
''',
logtype=LogType.ERROR,
timestamp=ourtime_new('2022-12-05 22:14:36')
))
logger.log(LogItemArgs(
cat='error-test',
log='''
aaa2
bbb2
''',
logtype=LogType.ERROR,
timestamp=ourtime_new('2022-12-05 22:14:36')
))
# Verify log directory exists
self.assertTrue(os.path.exists('/tmp/testlogs'), 'Log directory should exist')
# Get log file
files = ls('/tmp/testlogs')
self.assertEqual(len(files), 2) # Expecting two files: 2022-12-05-20.log and 2022-12-05-22.log
# Test search functionality
items_stdout = logger.search(
timestamp_from=ourtime_new('2022-11-01 20:14:35'),
timestamp_to=ourtime_new('2025-11-01 20:14:35'),
logtype=LogType.STDOUT
)
self.assertEqual(len(items_stdout), 2)
items_error = logger.search(
timestamp_from=ourtime_new('2022-11-01 20:14:35'),
timestamp_to=ourtime_new('2025-11-01 20:14:35'),
logtype=LogType.ERROR
)
self.assertEqual(len(items_error), 4)
# Test specific log content
found_error_log = False
for item in items_error:
if "This is an error\nWith details" in item.log:
found_error_log = True
break
self.assertTrue(found_error_log, "Expected error log content not found")
found_stdout_log = False
for item in items_stdout:
if "This is a test message\nWith a second line\nAnd a third line" in item.log:
found_stdout_log = True
break
self.assertTrue(found_stdout_log, "Expected stdout log content not found")
# Test search by category
items_test_app = logger.search(
timestamp_from=ourtime_new('2022-11-01 20:14:35'),
timestamp_to=ourtime_new('2025-11-01 20:14:35'),
cat='test-app'
)
self.assertEqual(len(items_test_app), 2)
items_error_test = logger.search(
timestamp_from=ourtime_new('2022-11-01 20:14:35'),
timestamp_to=ourtime_new('2025-11-01 20:14:35'),
cat='error-test'
)
self.assertEqual(len(items_error_test), 4)
# Test search by log content
items_with_aaa = logger.search(
timestamp_from=ourtime_new('2022-11-01 20:14:35'),
timestamp_to=ourtime_new('2025-11-01 20:14:35'),
log='aaa'
)
self.assertEqual(len(items_with_aaa), 2)
# Test search with timestamp range
items_specific_time = logger.search(
timestamp_from=ourtime_new('2022-12-05 22:00:00'),
timestamp_to=ourtime_new('2022-12-05 23:00:00'),
logtype=LogType.ERROR
)
self.assertEqual(len(items_specific_time), 2)
if __name__ == '__main__':
unittest.main()

72
lib/core/logger/model.py Normal file
View File

@ -0,0 +1,72 @@
from enum import Enum
from typing import Optional
from lib.data.ourtime.ourtime import OurTime
from lib.core.pathlib.pathlib import Path
class LogType(Enum):
STDOUT = "stdout"
ERROR = "error"
class LogItemArgs:
def __init__(self, cat: str, log: str, logtype: LogType, timestamp: Optional[OurTime] = None):
self.timestamp = timestamp
self.cat = cat
self.log = log
self.logtype = logtype
import os
from lib.core.texttools.texttools import name_fix, expand, dedent
from lib.data.ourtime.ourtime import OurTime, now as ourtime_now
class Logger:
def __init__(self, path: Path, lastlog_time: int = 0):
self.path = path
self.lastlog_time = lastlog_time
def log(self, args_: LogItemArgs):
args = args_
t = args.timestamp if args.timestamp else ourtime_now()
# Format category (max 10 chars, ascii only)
args.cat = name_fix(args.cat)
if len(args.cat) > 10:
raise ValueError('category cannot be longer than 10 chars')
args.cat = expand(args.cat, 10, ' ')
args.log = dedent(args.log).strip()
logfile_path = os.path.join(self.path.path, f"{t.dayhour()}.log")
# Create log file if it doesn't exist
if not os.path.exists(logfile_path):
with open(logfile_path, 'w') as f:
pass # Create empty file
self.lastlog_time = 0 # make sure we put time again
with open(logfile_path, 'a') as f:
content = ''
# Add timestamp if we're in a new second
if t.unix() > self.lastlog_time:
content += f"\n{t.time().format_ss()}\n"
self.lastlog_time = t.unix()
# Format log lines
error_prefix = 'E' if args.logtype == LogType.ERROR else ' '
lines = args.log.split('\n')
for i, line in enumerate(lines):
if i == 0:
content += f"{error_prefix} {args.cat} - {line}\n"
else:
content += f"{error_prefix} {line}\n"
f.write(content.rstrip()) # Use rstrip to remove trailing whitespace
f.write('\n') # Add a newline after each log entry for consistency
class LogItem:
def __init__(self, timestamp: OurTime, cat: str, log: str, logtype: LogType):
self.timestamp = timestamp
self.cat = cat
self.log = log
self.logtype = logtype

137
lib/core/logger/search.py Normal file
View File

@ -0,0 +1,137 @@
import os
from typing import Optional, List
from lib.core.texttools.texttools import name_fix
from lib.data.ourtime.ourtime import OurTime, new as ourtime_new
from lib.core.logger.model import Logger, LogItem, LogType
class SearchArgs:
def __init__(self, timestamp_from: Optional[OurTime] = None,
timestamp_to: Optional[OurTime] = None,
cat: str = "", log: str = "", logtype: Optional[LogType] = None,
maxitems: int = 10000):
self.timestamp_from = timestamp_from
self.timestamp_to = timestamp_to
self.cat = cat
self.log = log
self.logtype = logtype
self.maxitems = maxitems
def process(result: List[LogItem], current_item: LogItem, current_time: OurTime,
args: SearchArgs, from_time: int, to_time: int):
# Add previous item if it matches filters
log_epoch = current_item.timestamp.unix()
if log_epoch < from_time or log_epoch > to_time:
return
cat_match = (args.cat == '' or current_item.cat.strip() == args.cat)
log_match = (args.log == '' or args.log.lower() in current_item.log.lower())
logtype_match = (args.logtype is None or current_item.logtype == args.logtype)
if cat_match and log_match and logtype_match:
result.append(current_item)
def search(l: Logger, args_: SearchArgs) -> List[LogItem]:
args = args_
# Format category (max 10 chars, ascii only)
args.cat = name_fix(args.cat)
if len(args.cat) > 10:
raise ValueError('category cannot be longer than 10 chars')
timestamp_from = args.timestamp_from if args.timestamp_from else OurTime()
timestamp_to = args.timestamp_to if args.timestamp_to else OurTime()
# Get time range
from_time = timestamp_from.unix()
to_time = timestamp_to.unix()
if from_time > to_time:
raise ValueError(f'from_time cannot be after to_time: {from_time} < {to_time}')
result: List[LogItem] = []
# Find log files in time range
files = sorted(os.listdir(l.path.path))
for file in files:
if not file.endswith('.log'):
continue
# Parse dayhour from filename
dayhour = file[:-4] # remove .log
try:
file_time = ourtime_new(dayhour)
except ValueError:
continue # Skip if filename is not a valid time format
current_time = OurTime()
current_item = LogItem(OurTime(), "", "", LogType.STDOUT) # Initialize with dummy values
collecting = False
# Skip if file is outside time range
if file_time.unix() < from_time or file_time.unix() > to_time:
continue
# Read and parse log file
content = ""
try:
with open(os.path.join(l.path.path, file), 'r') as f:
content = f.read()
except FileNotFoundError:
continue
lines = content.split('\n')
for line in lines:
if len(result) >= args.maxitems:
return result
line_trim = line.strip()
if not line_trim:
continue
# Check if this is a timestamp line
if not (line.startswith(' ') or line.startswith('E')):
try:
current_time = ourtime_new(line_trim)
except ValueError:
continue # Skip if not a valid timestamp line
if collecting:
process(result, current_item, current_time, args, from_time, to_time)
collecting = False
continue
if collecting and len(line) > 14 and line[13] == '-':
process(result, current_item, current_time, args, from_time, to_time)
collecting = False
# Parse log line
is_error = line.startswith('E')
if not collecting:
# Start new item
cat_start = 2
cat_end = 12
log_start = 15
if len(line) < log_start:
continue # Line too short to contain log content
current_item = LogItem(
timestamp=current_time,
cat=line[cat_start:cat_end].strip(),
log=line[log_start:].strip(),
logtype=LogType.ERROR if is_error else LogType.STDOUT
)
collecting = True
else:
# Continuation line
if len(line_trim) < 16: # Check for minimum length for continuation line
current_item.log += '\n' + line_trim
else:
current_item.log += '\n' + line[15:].strip() # Use strip for continuation lines
# Add last item if collecting
if collecting:
process(result, current_item, current_time, args, from_time, to_time)
return result

0
lib/core/pathlib/__init__.py Normal file
View File

BIN
lib/core/pathlib/__pycache__/__init__.cpython-313.pyc Normal file
View File

Binary file not shown.

BIN
lib/core/pathlib/__pycache__/pathlib.cpython-313.pyc Normal file
View File

Binary file not shown.

80
lib/core/pathlib/pathlib.py Normal file
View File

@ -0,0 +1,80 @@
import os
class Path:
def __init__(self, path: str):
self.path = os.path.expanduser(path)
def exists(self) -> bool:
return os.path.exists(self.path)
def is_file(self) -> bool:
return os.path.isfile(self.path)
def is_dir(self) -> bool:
return os.path.isdir(self.path)
def read(self) -> str:
with open(self.path, 'r') as f:
return f.read()
def write(self, content: str):
os.makedirs(os.path.dirname(self.path), exist_ok=True)
with open(self.path, 'w') as f:
f.write(content)
def delete(self):
if self.is_file():
os.remove(self.path)
elif self.is_dir():
os.rmdir(self.path)
def list(self, recursive: bool = False, regex: list = None) -> list[str]:
files = []
if self.is_dir():
if recursive:
for root, _, filenames in os.walk(self.path):
for filename in filenames:
full_path = os.path.join(root, filename)
relative_path = os.path.relpath(full_path, self.path)
if regex:
import re
if any(re.match(r, relative_path) for r in regex):
files.append(relative_path)
else:
files.append(relative_path)
else:
for entry in os.listdir(self.path):
full_path = os.path.join(self.path, entry)
if os.path.isfile(full_path):
if regex:
import re
if any(re.match(r, entry) for r in regex):
files.append(entry)
else:
files.append(entry)
return files
def get(path: str) -> Path:
return Path(path)
def get_dir(path: str, create: bool = False) -> Path:
p = Path(path)
if create and not p.exists():
os.makedirs(p.path, exist_ok=True)
return p
def get_file(path: str, create: bool = False) -> Path:
p = Path(path)
if create and not p.exists():
os.makedirs(os.path.dirname(p.path), exist_ok=True)
with open(p.path, 'w') as f:
pass # Create empty file
return p
def rmdir_all(path: str):
if os.path.exists(path):
import shutil
shutil.rmtree(path)
def ls(path: str) -> list[str]:
return os.listdir(path)

0
lib/core/texttools/__init__.py Normal file
View File

142
lib/core/texttools/texttools.py Normal file
View File

@ -0,0 +1,142 @@
import re
def name_fix(name: str) -> str:
# VLang's name_fix converts '-' to '_' and cleans up special chars.
# Python's re.sub can handle this.
name = re.sub(r'[^a-zA-Z0-9_ ]', '', name.replace('-', '_'))
return name.strip()
def expand(txt: str, length: int, expand_with: str) -> str:
# Pads the string to the specified length.
return txt.ljust(length, expand_with)
def dedent(text: str) -> str:
# Removes common leading whitespace from every line.
# This is a simplified version of textwrap.dedent
lines = text.splitlines()
if not lines:
return ""
# Find the minimum indentation of non-empty lines
min_indent = float('inf')
for line in lines:
if line.strip():
indent = len(line) - len(line.lstrip())
min_indent = min(min_indent, indent)
if min_indent == float('inf'): # All lines are empty or just whitespace
return "\n".join([line.strip() for line in lines])
dedented_lines = [line[min_indent:] for line in lines]
return "\n".join(dedented_lines)
def remove_empty_lines(text: str) -> str:
lines = text.splitlines()
return "\n".join([line for line in lines if line.strip()])
def remove_double_lines(text: str) -> str:
lines = text.splitlines()
cleaned_lines = []
prev_empty = False
for line in lines:
is_empty = not line.strip()
if is_empty and prev_empty:
continue
cleaned_lines.append(line)
prev_empty = is_empty
return "\n".join(cleaned_lines)
def ascii_clean(r: str) -> str:
return r.encode('ascii', 'ignore').decode('ascii')
def name_clean(r: str) -> str:
return re.sub(r'[^a-zA-Z0-9]', '', r)
def name_fix_keepspace(name_: str) -> str:
# Similar to name_fix but keeps spaces.
return re.sub(r'[^a-zA-Z0-9 ]', '', name_.replace('-', '_')).strip()
def name_fix_no_ext(name_: str) -> str:
return os.path.splitext(name_)[0]
def name_fix_snake_to_pascal(name: str) -> str:
return ''.join(word.capitalize() for word in name.split('_'))
def snake_case(name: str) -> str:
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
def name_split(name: str) -> tuple[str, str]:
parts = name.split('.')
if len(parts) > 1:
return parts[0], '.'.join(parts[1:])
return name, ""
def cmd_line_args_parser(text: str) -> list[str]:
# A simple parser, might need more robust solution for complex cases
import shlex
return shlex.split(text)
def text_remove_quotes(text: str) -> str:
return re.sub(r'["\'].*?["\']', '', text)
def check_exists_outside_quotes(text: str, items: list[str]) -> bool:
# This is a simplified implementation. A full implementation would require
# more complex parsing to correctly identify text outside quotes.
cleaned_text = text_remove_quotes(text)
for item in items:
if item in cleaned_text:
return True
return False
def is_int(text: str) -> bool:
return text.isdigit()
def is_upper_text(text: str) -> bool:
return text.isupper() and text.isalpha()
def multiline_to_single(text: str) -> str:
return text.replace('\n', '\\n').replace('\r', '')
def split_smart(t: str, delimiter_: str) -> list[str]:
# This is a placeholder, a smart split would need to handle quotes and escapes
return t.split(delimiter_)
def version(text_: str) -> int:
# Converts version strings like "v0.4.36" to 4036 or "v1.4.36" to 1004036
match = re.match(r'v?(\d+)\.(\d+)\.(\d+)', text_)
if match:
major, minor, patch = int(match.group(1)), int(match.group(2)), int(match.group(3))
if major == 0:
return minor * 100 + patch
else:
return major * 1000000 + minor * 100 + patch
return 0
def format_rfc1123(dt: datetime) -> str:
return dt.strftime('%a, %d %b %Y %H:%M:%S GMT')
def to_array(r: str) -> list[str]:
if ',' in r:
return [item.strip() for item in r.split(',')]
return [item.strip() for item in r.splitlines() if item.strip()]
def to_array_int(r: str) -> list[int]:
return [int(item) for item in to_array(r) if item.isdigit()]
def to_map(mapstring: str, line: str, delimiter_: str = ' ') -> dict[str, str]:
# This is a simplified implementation. The VLang version is more complex.
# It assumes a space delimiter for now.
keys = [k.strip() for k in mapstring.split(',')]
values = line.split(delimiter_)
result = {}
val_idx = 0
for key in keys:
if key == '-':
val_idx += 1
continue
if val_idx < len(values):
result[key] = values[val_idx]
val_idx += 1
return result

0
lib/data/ourtime/__init__.py Normal file
View File

BIN
lib/data/ourtime/__pycache__/__init__.cpython-313.pyc Normal file
View File

Binary file not shown.

BIN
lib/data/ourtime/__pycache__/ourtime.cpython-313.pyc Normal file
View File

Binary file not shown.

123
lib/data/ourtime/ourtime.py Normal file
View File

@ -0,0 +1,123 @@
from datetime import datetime, timedelta
import re
class OurTime:
def __init__(self, dt: datetime = None):
self._dt = dt if dt else datetime.min
def __str__(self) -> str:
return self.str()
def str(self) -> str:
if self._dt == datetime.min:
return ""
return self._dt.strftime('%Y-%m-%d %H:%M')
def day(self) -> str:
if self._dt == datetime.min:
return ""
return self._dt.strftime('%Y-%m-%d')
def key(self) -> str:
if self._dt == datetime.min:
return ""
return self._dt.strftime('%Y_%m_%d_%H_%M_%S')
def md(self) -> str:
if self._dt == datetime.min:
return ""
return self._dt.strftime('%Y-%m-%d %H:%M:%S')
def unix(self) -> int:
if self._dt == datetime.min:
return 0
return int(self._dt.timestamp())
def empty(self) -> bool:
return self._dt == datetime.min
def dayhour(self) -> str:
if self._dt == datetime.min:
return ""
return self._dt.strftime('%Y-%m-%d-%H')
def time(self):
# This is a simplified representation, as VLang's time() returns a time object.
# Here, we return self to allow chaining format_ss().
return self
def format_ss(self) -> str:
if self._dt == datetime.min:
return ""
return self._dt.strftime('%H:%M:%S')
def warp(self, expression: str):
if self._dt == datetime.min:
return
parts = expression.split()
for part in parts:
match = re.match(r'([+-]?\d+)([smhdwMQY])', part)
if not match:
continue
value = int(match.group(1))
unit = match.group(2)
if unit == 's':
self._dt += timedelta(seconds=value)
elif unit == 'm':
self._dt += timedelta(minutes=value)
elif unit == 'h':
self._dt += timedelta(hours=value)
elif unit == 'd':
self._dt += timedelta(days=value)
elif unit == 'w':
self._dt += timedelta(weeks=value)
elif unit == 'M':
# Approximate months, for more accuracy, a proper dateutil.relativedelta would be needed
self._dt += timedelta(days=value * 30)
elif unit == 'Q':
self._dt += timedelta(days=value * 90)
elif unit == 'Y':
self._dt += timedelta(days=value * 365)
def now() -> OurTime:
return OurTime(datetime.now())
def new(time_str: str) -> OurTime:
if not time_str:
return OurTime()
formats = [
'%Y-%m-%d %H:%M:%S',
'%Y-%m-%d %H:%M',
'%Y-%m-%d %H',
'%Y-%m-%d',
'%d-%m-%Y %H:%M:%S',
'%d-%m-%Y %H:%M',
'%d-%m-%Y %H',
'%d-%m-%Y',
'%H:%M:%S', # For time() and format_ss() usage
]
for fmt in formats:
try:
dt = datetime.strptime(time_str, fmt)
return OurTime(dt)
except ValueError:
pass
# Handle relative time expressions
try:
# Create a dummy OurTime object to use its warp method
temp_time = now()
temp_time.warp(time_str)
return temp_time
except Exception:
pass
raise ValueError(f"Could not parse time string: {time_str}")
def new_from_epoch(epoch: int) -> OurTime:
return OurTime(datetime.fromtimestamp(epoch))