diff --git a/lib/core/logger/__init__.py b/lib/core/logger/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/core/logger/__pycache__/__init__.cpython-313.pyc b/lib/core/logger/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..f3f307d Binary files /dev/null and b/lib/core/logger/__pycache__/__init__.cpython-313.pyc differ diff --git a/lib/core/logger/__pycache__/factory.cpython-313.pyc b/lib/core/logger/__pycache__/factory.cpython-313.pyc new file mode 100644 index 0000000..1557e79 Binary files /dev/null and b/lib/core/logger/__pycache__/factory.cpython-313.pyc differ diff --git a/lib/core/logger/__pycache__/model.cpython-313.pyc b/lib/core/logger/__pycache__/model.cpython-313.pyc new file mode 100644 index 0000000..94e3cf7 Binary files /dev/null and b/lib/core/logger/__pycache__/model.cpython-313.pyc differ diff --git a/lib/core/logger/factory.py b/lib/core/logger/factory.py new file mode 100644 index 0000000..a101936 --- /dev/null +++ b/lib/core/logger/factory.py @@ -0,0 +1,9 @@ +from lib.core.pathlib.pathlib import get_dir +from lib.core.logger.model import Logger + +def new(path: str) -> Logger: + p = get_dir(path=path, create=True) + return Logger( + path=p, + lastlog_time=0 + ) \ No newline at end of file diff --git a/lib/core/logger/instructions.md b/lib/core/logger/instructions.md new file mode 100644 index 0000000..1af395d --- /dev/null +++ b/lib/core/logger/instructions.md @@ -0,0 +1,822 @@ + +/Users/despiegk/code/github/freeflowuniverse/herolib +├── aiprompts +│ └── herolib_core +│ ├── core_ourtime.md +│ ├── core_paths.md +│ └── core_text.md +└── lib + └── core + └── logger + ├── factory.v + ├── log_test.v + ├── log.v + ├── model.v + ├── readme.md + └── search.v + + + + +File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/factory.v +```v +module logger + +import freeflowuniverse.herolib.core.pathlib + +pub fn new(path string) !Logger { + mut p := pathlib.get_dir(path: path, create: true)! + return Logger{ + path: p + lastlog_time: 0 + } +} + +``` + +File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/log_test.v +```v +module logger + +import os +import freeflowuniverse.herolib.data.ourtime +import freeflowuniverse.herolib.core.pathlib + +fn testsuite_begin() { + if os.exists('/tmp/testlogs') { + os.rmdir_all('/tmp/testlogs')! + } +} + +fn test_logger() { + mut logger := new('/tmp/testlogs')! + + // Test stdout logging + logger.log(LogItemArgs{ + cat: 'test-app' + log: 'This is a test message\nWith a second line\nAnd a third line' + logtype: .stdout + timestamp: ourtime.new('2022-12-05 20:14:35')! + })! + + // Test error logging + logger.log(LogItemArgs{ + cat: 'error-test' + log: 'This is an error\nWith details' + logtype: .error + timestamp: ourtime.new('2022-12-05 20:14:35')! + })! + + logger.log(LogItemArgs{ + cat: 'test-app' + log: 'This is a test message\nWith a second line\nAnd a third line' + logtype: .stdout + timestamp: ourtime.new('2022-12-05 20:14:36')! + })! + + logger.log(LogItemArgs{ + cat: 'error-test' + log: ' + This is an error + + With details + ' + logtype: .error + timestamp: ourtime.new('2022-12-05 20:14:36')! + })! + + logger.log(LogItemArgs{ + cat: 'error-test' + log: ' + aaa + + bbb + ' + logtype: .error + timestamp: ourtime.new('2022-12-05 22:14:36')! + })! + + logger.log(LogItemArgs{ + cat: 'error-test' + log: ' + aaa2 + + bbb2 + ' + logtype: .error + timestamp: ourtime.new('2022-12-05 22:14:36')! + })! + + // Verify log directory exists + assert os.exists('/tmp/testlogs'), 'Log directory should exist' + + // Get log file + files := os.ls('/tmp/testlogs')! + assert files.len == 2 + + mut file := pathlib.get_file( + path: '/tmp/testlogs/${files[0]}' + create: false + )! + + content := file.read()!.trim_space() + + items_stdout := logger.search( + timestamp_from: ourtime.new('2022-11-1 20:14:35')! + timestamp_to: ourtime.new('2025-11-1 20:14:35')! + logtype: .stdout + )! + assert items_stdout.len == 2 + + items_error := logger.search( + timestamp_from: ourtime.new('2022-11-1 20:14:35')! + timestamp_to: ourtime.new('2025-11-1 20:14:35')! + logtype: .error + )! + assert items_error.len == 4 +} + +fn testsuite_end() { + // if os.exists('/tmp/testlogs') { + // os.rmdir_all('/tmp/testlogs')! + // } +} + +``` + +File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/log.v +```v +module logger + +import os +import freeflowuniverse.herolib.core.texttools +import freeflowuniverse.herolib.data.ourtime + +@[params] +pub struct LogItemArgs { +pub mut: + timestamp ?ourtime.OurTime + cat string + log string + logtype LogType +} + +pub fn (mut l Logger) log(args_ LogItemArgs) ! { + mut args := args_ + + t := args.timestamp or { + t2 := ourtime.now() + t2 + } + + // Format category (max 10 chars, ascii only) + args.cat = texttools.name_fix(args.cat) + if args.cat.len > 10 { + return error('category cannot be longer than 10 chars') + } + args.cat = texttools.expand(args.cat, 10, ' ') + + args.log = texttools.dedent(args.log).trim_space() + + mut logfile_path := '${l.path.path}/${t.dayhour()}.log' + + // Create log file if it doesn't exist + if !os.exists(logfile_path) { + os.write_file(logfile_path, '')! + l.lastlog_time = 0 // make sure we put time again + } + + mut f := os.open_append(logfile_path)! + + mut content := '' + + // Add timestamp if we're in a new second + if t.unix() > l.lastlog_time { + content += '\n${t.time().format_ss()}\n' + l.lastlog_time = t.unix() + } + + // Format log lines + error_prefix := if args.logtype == .error { 'E' } else { ' ' } + lines := args.log.split('\n') + + for i, line in lines { + if i == 0 { + content += '${error_prefix} ${args.cat} - ${line}\n' + } else { + content += '${error_prefix} ${line}\n' + } + } + f.writeln(content.trim_space_right())! + f.close() +} + +``` + +File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/model.v +```v +module logger + +import freeflowuniverse.herolib.data.ourtime +import freeflowuniverse.herolib.core.pathlib + +@[heap] +pub struct Logger { +pub mut: + path pathlib.Path + lastlog_time i64 // to see in log format, every second we put a time down, we need to know if we are in a new second (logs can come in much faster) +} + +pub struct LogItem { +pub mut: + timestamp ourtime.OurTime + cat string + log string + logtype LogType +} + +pub enum LogType { + stdout + error +} + +``` + +File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/readme.md +```md +# Logger Module + +A simple logging system that provides structured logging with search capabilities. + +Logs are stored in hourly files with a consistent format that makes them both human-readable and machine-parseable. + +## Features + +- Structured logging with categories and error types +- Automatic timestamp management +- Multi-line message support +- Search functionality with filtering options +- Human-readable log format + +## Usage + +```v +import freeflowuniverse.herolib.core.logger +import freeflowuniverse.herolib.data.ourtime + +// Create a new logger +mut l := logger.new(path: '/var/logs')! + +// Log a message +l.log( + cat: 'system', + log: 'System started successfully', + logtype: .stdout +)! + +// Log an error +l.log( + cat: 'system', + log: 'Failed to connect\nRetrying in 5 seconds...', + logtype: .error +)! + +// Search logs +results := l.search( + timestamp_from: ourtime.now().warp("-24h"), // Last 24 hours + cat: 'system', // Filter by category + log: 'failed', // Search in message content + logtype: .error, // Only error messages + maxitems: 100 // Limit results +)! +``` + +## Log Format + +Each log file is named using the format `YYYY-MM-DD-HH.log` and contains entries in the following format: + +``` +21:23:42 + system - This is a normal log message + system - This is a multi-line message + second line with proper indentation + third line maintaining alignment +E error_cat - This is an error message +E second line of error +E third line of error +``` + +### Format Rules + +- Time stamps (HH:MM:SS) are written once per second when the log time changes +- Categories are: + - Limited to 10 characters maximum + - Padded with spaces to exactly 10 characters + - Any `-` in category names are converted to `_` +- Each line starts with either: + - ` ` (space) for normal logs (LogType.stdout) + - `E` for error logs (LogType.error) +- Multi-line messages maintain consistent indentation (14 spaces after the prefix) + +``` + +File: /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger/search.v +```v +module logger + +import os +import freeflowuniverse.herolib.core.texttools +import freeflowuniverse.herolib.data.ourtime + +@[params] +pub struct SearchArgs { +pub mut: + timestamp_from ?ourtime.OurTime + timestamp_to ?ourtime.OurTime + cat string // can be empty + log string // any content in here will be looked for + logtype LogType + maxitems int = 10000 +} + +pub fn (mut l Logger) search(args_ SearchArgs) ![]LogItem { + mut args := args_ + + // Format category (max 10 chars, ascii only) + args.cat = texttools.name_fix(args.cat) + if args.cat.len > 10 { + return error('category cannot be longer than 10 chars') + } + + mut timestamp_from := args.timestamp_from or { ourtime.OurTime{} } + mut timestamp_to := args.timestamp_to or { ourtime.OurTime{} } + + // Get time range + from_time := timestamp_from.unix() + to_time := timestamp_to.unix() + if from_time > to_time { + return error('from_time cannot be after to_time: ${from_time} < ${to_time}') + } + + mut result := []LogItem{} + + // Find log files in time range + mut files := os.ls(l.path.path)! + files.sort() + + for file in files { + if !file.ends_with('.log') { + continue + } + + // Parse dayhour from filename + dayhour := file[..file.len - 4] // remove .log + file_time := ourtime.new(dayhour)! + mut current_time := ourtime.OurTime{} + mut current_item := LogItem{} + mut collecting := false + + // Skip if file is outside time range + if file_time.unix() < from_time || file_time.unix() > to_time { + continue + } + + // Read and parse log file + content := os.read_file('${l.path.path}/${file}')! + lines := content.split('\n') + + for line in lines { + if result.len >= args.maxitems { + return result + } + + line_trim := line.trim_space() + if line_trim == '' { + continue + } + + // Check if this is a timestamp line + if !(line.starts_with(' ') || line.starts_with('E')) { + current_time = ourtime.new(line_trim)! + if collecting { + process(mut result, current_item, current_time, args, from_time, to_time)! + } + collecting = false + continue + } + + if collecting && line.len > 14 && line[13] == `-` { + process(mut result, current_item, current_time, args, from_time, to_time)! + collecting = false + } + + // Parse log line + is_error := line.starts_with('E') + if !collecting { + // Start new item + current_item = LogItem{ + timestamp: current_time + cat: line[2..12].trim_space() + log: line[15..].trim_space() + logtype: if is_error { .error } else { .stdout } + } + // println('new current item: ${current_item}') + collecting = true + } else { + // Continuation line + if line_trim.len < 16 { + current_item.log += '\n' + } else { + current_item.log += '\n' + line[15..] + } + } + } + + // Add last item if collecting + if collecting { + process(mut result, current_item, current_time, args, from_time, to_time)! + } + } + + return result +} + +fn process(mut result []LogItem, current_item LogItem, current_time ourtime.OurTime, args SearchArgs, from_time i64, to_time i64) ! { + // Add previous item if it matches filters + log_epoch := current_item.timestamp.unix() + if log_epoch < from_time || log_epoch > to_time { + return + } + if (args.cat == '' || current_item.cat.trim_space() == args.cat) + && (args.log == '' || current_item.log.contains(args.log)) + && args.logtype == current_item.logtype { + result << current_item + } +} + +``` + +File: /Users/despiegk/code/github/freeflowuniverse/herolib/aiprompts/herolib_core/core_ourtime.md +```md +# OurTime Module + +The `OurTime` module in V provides flexible time handling, supporting relative and absolute time formats, Unix timestamps, and formatting utilities. + +## Key Features +- Create time objects from strings or current time +- Relative time expressions (e.g., `+1h`, `-2d`) +- Absolute time formats (e.g., `YYYY-MM-DD HH:mm:ss`) +- Unix timestamp conversion +- Time formatting and warping + +## Basic Usage + +```v +import freeflowuniverse.herolib.data.ourtime + +// Current time +mut t := ourtime.now() + +// From string +t2 := ourtime.new('2022-12-05 20:14:35')! + +// Get formatted string +println(t2.str()) // e.g., 2022-12-05 20:14 + +// Get Unix timestamp +println(t2.unix()) // e.g., 1670271275 +``` + +## Time Formats + +### Relative Time + +Use `s` (seconds), `h` (hours), `d` (days), `w` (weeks), `M` (months), `Q` (quarters), `Y` (years). + +```v +// Create with relative time +mut t := ourtime.new('+1w +2d -4h')! + +// Warp existing time +mut t2 := ourtime.now() +t2.warp('+1h')! +``` + +### Absolute Time + +Supports `YYYY-MM-DD HH:mm:ss`, `YYYY-MM-DD HH:mm`, `YYYY-MM-DD HH`, `YYYY-MM-DD`, `DD-MM-YYYY`. + +```v +t1 := ourtime.new('2022-12-05 20:14:35')! +t2 := ourtime.new('2022-12-05')! // Time defaults to 00:00:00 +``` + +## Methods Overview + +### Creation + +```v +now_time := ourtime.now() +from_string := ourtime.new('2023-01-15')! +from_epoch := ourtime.new_from_epoch(1673788800) +``` + +### Formatting + +```v +mut t := ourtime.now() +println(t.str()) // YYYY-MM-DD HH:mm +println(t.day()) // YYYY-MM-DD +println(t.key()) // YYYY_MM_DD_HH_mm_ss +println(t.md()) // Markdown format +``` + +### Operations + +```v +mut t := ourtime.now() +t.warp('+1h')! // Move 1 hour forward +unix_ts := t.unix() +is_empty := t.empty() +``` + +## Error Handling + +Time parsing methods return a `Result` type and should be handled with `!` or `or` blocks. + +```v +t_valid := ourtime.new('2023-01-01')! +t_invalid := ourtime.new('bad-date') or { + println('Error: ${err}') + ourtime.now() // Fallback +} + +``` + +File: /Users/despiegk/code/github/freeflowuniverse/herolib/aiprompts/herolib_core/core_paths.md +```md +# Pathlib Usage Guide + +## Overview + +The pathlib module provides a comprehensive interface for handling file system operations. Key features include: + +- Robust path handling for files, directories, and symlinks +- Support for both absolute and relative paths +- Automatic home directory expansion (~) +- Recursive directory operations +- Path filtering and listing +- File and directory metadata access + +## Basic Usage + +### Importing pathlib +```v +import freeflowuniverse.herolib.core.pathlib +``` + +### Creating Path Objects +```v +// Create a Path object for a file +mut file_path := pathlib.get("path/to/file.txt") + +// Create a Path object for a directory +mut dir_path := pathlib.get("path/to/directory") +``` + +### Basic Path Operations +```v +// Get absolute path +abs_path := file_path.absolute() + +// Get real path (resolves symlinks) +real_path := file_path.realpath() + +// Check if path exists +if file_path.exists() { + // Path exists +} +``` + +## Path Properties and Methods + +### Path Types +```v +// Check if path is a file +if file_path.is_file() { + // Handle as file +} + +// Check if path is a directory +if dir_path.is_dir() { + // Handle as directory +} + +// Check if path is a symlink +if file_path.is_link() { + // Handle as symlink +} +``` + +### Path Normalization +```v +// Normalize path (remove extra slashes, resolve . and ..) +normalized_path := file_path.path_normalize() + +// Get path directory +dir_path := file_path.path_dir() + +// Get path name without extension +name_no_ext := file_path.name_no_ext() +``` + +## File and Directory Operations + +### File Operations +```v +// Write to file +file_path.write("Content to write")! + +// Read from file +content := file_path.read()! + +// Delete file +file_path.delete()! +``` + +### Directory Operations +```v +// Create directory +mut dir := pathlib.get_dir( + path: "path/to/new/dir" + create: true +)! + +// List directory contents +mut dir_list := dir.list()! + +// Delete directory +dir.delete()! +``` + +### Symlink Operations +```v +// Create symlink +file_path.link("path/to/symlink", delete_exists: true)! + +// Resolve symlink +real_path := file_path.realpath() +``` + +## Advanced Operations + +### Path Copying +```v +// Copy file to destination +file_path.copy(dest: "path/to/destination")! +``` + +### Recursive Operations +```v +// List directory recursively +mut recursive_list := dir.list(recursive: true)! + +// Delete directory recursively +dir.delete()! +``` + +### Path Filtering +```v +// List files matching pattern +mut filtered_list := dir.list( + regex: [r".*\.txt$"], + recursive: true +)! +``` + +## Best Practices + +### Error Handling +```v +if file_path.exists() { + // Safe to operate +} else { + // Handle missing file +} +``` + + +``` + +File: /Users/despiegk/code/github/freeflowuniverse/herolib/aiprompts/herolib_core/core_text.md +```md +# TextTools Module + +The `texttools` module provides a comprehensive set of utilities for text manipulation and processing. + +## Functions and Examples: + +```v +import freeflowuniverse.herolib.core.texttools + +assert hello_world == texttools.name_fix("Hello World!") + +``` +### Name/Path Processing +* `name_fix(name string) string`: Normalizes filenames and paths. +* `name_fix_keepspace(name string) !string`: Like name_fix but preserves spaces. +* `name_fix_no_ext(name_ string) string`: Removes file extension. +* `name_fix_snake_to_pascal(name string) string`: Converts snake_case to PascalCase. + ```v + name := texttools.name_fix_snake_to_pascal("hello_world") // Result: "HelloWorld" + ``` +* `snake_case(name string) string`: Converts PascalCase to snake_case. + ```v + name := texttools.snake_case("HelloWorld") // Result: "hello_world" + ``` +* `name_split(name string) !(string, string)`: Splits name into site and page components. + + +### Text Cleaning +* `name_clean(r string) string`: Normalizes names by removing special characters. + ```v + name := texttools.name_clean("Hello@World!") // Result: "HelloWorld" + ``` +* `ascii_clean(r string) string`: Removes all non-ASCII characters. +* `remove_empty_lines(text string) string`: Removes empty lines from text. + ```v + text := texttools.remove_empty_lines("line1\n\nline2\n\n\nline3") // Result: "line1\nline2\nline3" + ``` +* `remove_double_lines(text string) string`: Removes consecutive empty lines. +* `remove_empty_js_blocks(text string) string`: Removes empty code blocks (```...```). + +### Command Line Parsing +* `cmd_line_args_parser(text string) ![]string`: Parses command line arguments with support for quotes and escaping. + ```v + args := texttools.cmd_line_args_parser("'arg with spaces' --flag=value") // Result: ['arg with spaces', '--flag=value'] + ``` +* `text_remove_quotes(text string) string`: Removes quoted sections from text. +* `check_exists_outside_quotes(text string, items []string) bool`: Checks if items exist in text outside of quotes. + +### Text Expansion +* `expand(txt_ string, l int, expand_with string) string`: Expands text to a specified length with a given character. + +### Indentation +* `indent(text string, prefix string) string`: Adds indentation prefix to each line. + ```v + text := texttools.indent("line1\nline2", " ") // Result: " line1\n line2\n" + ``` +* `dedent(text string) string`: Removes common leading whitespace from every line. + ```v + text := texttools.dedent(" line1\n line2") // Result: "line1\nline2" + ``` + +### String Validation +* `is_int(text string) bool`: Checks if text contains only digits. +* `is_upper_text(text string) bool`: Checks if text contains only uppercase letters. + +### Multiline Processing +* `multiline_to_single(text string) !string`: Converts multiline text to a single line with proper escaping. + +### Text Splitting +* `split_smart(t string, delimiter_ string) []string`: Intelligent string splitting that respects quotes. + +### Tokenization +* `tokenize(text_ string) TokenizerResult`: Tokenizes text into meaningful parts. +* `text_token_replace(text string, tofind string, replacewith string) !string`: Replaces tokens in text. + +### Version Parsing +* `version(text_ string) int`: Converts version strings to comparable integers. + ```v + ver := texttools.version("v0.4.36") // Result: 4036 + ver = texttools.version("v1.4.36") // Result: 1004036 + ``` + +### Formatting +* `format_rfc1123(t time.Time) string`: Formats a time.Time object into RFC 1123 format. + + +### Array Operations +* `to_array(r string) []string`: Converts a comma or newline separated list to an array of strings. + ```v + text := "item1,item2,item3" + array := texttools.to_array(text) // Result: ['item1', 'item2', 'item3'] + ``` +* `to_array_int(r string) []int`: Converts a text list to an array of integers. +* `to_map(mapstring string, line string, delimiter_ string) map[string]string`: Intelligent mapping of a line to a map based on a template. + ```v + r := texttools.to_map("name,-,-,-,-,pid,-,-,-,-,path", + "root 304 0.0 0.0 408185328 1360 ?? S 16Dec23 0:34.06 /usr/sbin/distnoted") + // Result: {'name': 'root', 'pid': '1360', 'path': '/usr/sbin/distnoted'} + ``` + +``` + + +create a module in python in location lib/core/logger in herolib_python +which reimplements /Users/despiegk/code/github/freeflowuniverse/herolib/lib/core/logger +all features need to be reimplemented + + +write me an implementation plan for my coding agent + \ No newline at end of file diff --git a/lib/core/logger/log.py b/lib/core/logger/log.py new file mode 100644 index 0000000..64787c5 --- /dev/null +++ b/lib/core/logger/log.py @@ -0,0 +1,3 @@ +# This file is now empty as the log function has been moved to model.py +# It can be removed or kept as a placeholder if needed for future extensions. +# For now, we will keep it empty. \ No newline at end of file diff --git a/lib/core/logger/log_test.py b/lib/core/logger/log_test.py new file mode 100644 index 0000000..278b3a9 --- /dev/null +++ b/lib/core/logger/log_test.py @@ -0,0 +1,150 @@ +import unittest +import os +import shutil +from lib.core.logger.factory import new +from lib.core.logger.model import LogItemArgs, LogType, Logger # Import Logger class +from lib.data.ourtime.ourtime import new as ourtime_new, now as ourtime_now +from lib.core.pathlib.pathlib import get_file, ls, rmdir_all + +class TestLogger(unittest.TestCase): + def setUp(self): + # Corresponds to testsuite_begin() + if os.path.exists('/tmp/testlogs'): + rmdir_all('/tmp/testlogs') + + def tearDown(self): + # Corresponds to testsuite_end() + # if os.path.exists('/tmp/testlogs'): + # rmdir_all('/tmp/testlogs') + pass + + def test_logger_functionality(self): + logger = new('/tmp/testlogs') + + # Test stdout logging + logger.log(LogItemArgs( + cat='test-app', + log='This is a test message\nWith a second line\nAnd a third line', + logtype=LogType.STDOUT, + timestamp=ourtime_new('2022-12-05 20:14:35') + )) + + # Test error logging + logger.log(LogItemArgs( + cat='error-test', + log='This is an error\nWith details', + logtype=LogType.ERROR, + timestamp=ourtime_new('2022-12-05 20:14:35') + )) + + logger.log(LogItemArgs( + cat='test-app', + log='This is a test message\nWith a second line\nAnd a third line', + logtype=LogType.STDOUT, + timestamp=ourtime_new('2022-12-05 20:14:36') + )) + + logger.log(LogItemArgs( + cat='error-test', + log=''' + This is an error + + With details + ''', + logtype=LogType.ERROR, + timestamp=ourtime_new('2022-12-05 20:14:36') + )) + + logger.log(LogItemArgs( + cat='error-test', + log=''' + aaa + + bbb + ''', + logtype=LogType.ERROR, + timestamp=ourtime_new('2022-12-05 22:14:36') + )) + + logger.log(LogItemArgs( + cat='error-test', + log=''' + aaa2 + + bbb2 + ''', + logtype=LogType.ERROR, + timestamp=ourtime_new('2022-12-05 22:14:36') + )) + + # Verify log directory exists + self.assertTrue(os.path.exists('/tmp/testlogs'), 'Log directory should exist') + + # Get log file + files = ls('/tmp/testlogs') + self.assertEqual(len(files), 2) # Expecting two files: 2022-12-05-20.log and 2022-12-05-22.log + + # Test search functionality + items_stdout = logger.search( + timestamp_from=ourtime_new('2022-11-01 20:14:35'), + timestamp_to=ourtime_new('2025-11-01 20:14:35'), + logtype=LogType.STDOUT + ) + self.assertEqual(len(items_stdout), 2) + + items_error = logger.search( + timestamp_from=ourtime_new('2022-11-01 20:14:35'), + timestamp_to=ourtime_new('2025-11-01 20:14:35'), + logtype=LogType.ERROR + ) + self.assertEqual(len(items_error), 4) + + # Test specific log content + found_error_log = False + for item in items_error: + if "This is an error\nWith details" in item.log: + found_error_log = True + break + self.assertTrue(found_error_log, "Expected error log content not found") + + found_stdout_log = False + for item in items_stdout: + if "This is a test message\nWith a second line\nAnd a third line" in item.log: + found_stdout_log = True + break + self.assertTrue(found_stdout_log, "Expected stdout log content not found") + + # Test search by category + items_test_app = logger.search( + timestamp_from=ourtime_new('2022-11-01 20:14:35'), + timestamp_to=ourtime_new('2025-11-01 20:14:35'), + cat='test-app' + ) + self.assertEqual(len(items_test_app), 2) + + items_error_test = logger.search( + timestamp_from=ourtime_new('2022-11-01 20:14:35'), + timestamp_to=ourtime_new('2025-11-01 20:14:35'), + cat='error-test' + ) + self.assertEqual(len(items_error_test), 4) + + # Test search by log content + items_with_aaa = logger.search( + timestamp_from=ourtime_new('2022-11-01 20:14:35'), + timestamp_to=ourtime_new('2025-11-01 20:14:35'), + log='aaa' + ) + self.assertEqual(len(items_with_aaa), 2) + + # Test search with timestamp range + items_specific_time = logger.search( + timestamp_from=ourtime_new('2022-12-05 22:00:00'), + timestamp_to=ourtime_new('2022-12-05 23:00:00'), + logtype=LogType.ERROR + ) + self.assertEqual(len(items_specific_time), 2) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/lib/core/logger/model.py b/lib/core/logger/model.py new file mode 100644 index 0000000..3467811 --- /dev/null +++ b/lib/core/logger/model.py @@ -0,0 +1,72 @@ +from enum import Enum +from typing import Optional +from lib.data.ourtime.ourtime import OurTime +from lib.core.pathlib.pathlib import Path + +class LogType(Enum): + STDOUT = "stdout" + ERROR = "error" + +class LogItemArgs: + def __init__(self, cat: str, log: str, logtype: LogType, timestamp: Optional[OurTime] = None): + self.timestamp = timestamp + self.cat = cat + self.log = log + self.logtype = logtype + +import os +from lib.core.texttools.texttools import name_fix, expand, dedent +from lib.data.ourtime.ourtime import OurTime, now as ourtime_now + +class Logger: + def __init__(self, path: Path, lastlog_time: int = 0): + self.path = path + self.lastlog_time = lastlog_time + + def log(self, args_: LogItemArgs): + args = args_ + + t = args.timestamp if args.timestamp else ourtime_now() + + # Format category (max 10 chars, ascii only) + args.cat = name_fix(args.cat) + if len(args.cat) > 10: + raise ValueError('category cannot be longer than 10 chars') + args.cat = expand(args.cat, 10, ' ') + + args.log = dedent(args.log).strip() + + logfile_path = os.path.join(self.path.path, f"{t.dayhour()}.log") + + # Create log file if it doesn't exist + if not os.path.exists(logfile_path): + with open(logfile_path, 'w') as f: + pass # Create empty file + self.lastlog_time = 0 # make sure we put time again + + with open(logfile_path, 'a') as f: + content = '' + + # Add timestamp if we're in a new second + if t.unix() > self.lastlog_time: + content += f"\n{t.time().format_ss()}\n" + self.lastlog_time = t.unix() + + # Format log lines + error_prefix = 'E' if args.logtype == LogType.ERROR else ' ' + lines = args.log.split('\n') + + for i, line in enumerate(lines): + if i == 0: + content += f"{error_prefix} {args.cat} - {line}\n" + else: + content += f"{error_prefix} {line}\n" + f.write(content.rstrip()) # Use rstrip to remove trailing whitespace + f.write('\n') # Add a newline after each log entry for consistency + +class LogItem: + def __init__(self, timestamp: OurTime, cat: str, log: str, logtype: LogType): + self.timestamp = timestamp + self.cat = cat + self.log = log + self.logtype = logtype \ No newline at end of file diff --git a/lib/core/logger/search.py b/lib/core/logger/search.py new file mode 100644 index 0000000..a497236 --- /dev/null +++ b/lib/core/logger/search.py @@ -0,0 +1,137 @@ +import os +from typing import Optional, List +from lib.core.texttools.texttools import name_fix +from lib.data.ourtime.ourtime import OurTime, new as ourtime_new +from lib.core.logger.model import Logger, LogItem, LogType + +class SearchArgs: + def __init__(self, timestamp_from: Optional[OurTime] = None, + timestamp_to: Optional[OurTime] = None, + cat: str = "", log: str = "", logtype: Optional[LogType] = None, + maxitems: int = 10000): + self.timestamp_from = timestamp_from + self.timestamp_to = timestamp_to + self.cat = cat + self.log = log + self.logtype = logtype + self.maxitems = maxitems + +def process(result: List[LogItem], current_item: LogItem, current_time: OurTime, + args: SearchArgs, from_time: int, to_time: int): + # Add previous item if it matches filters + log_epoch = current_item.timestamp.unix() + if log_epoch < from_time or log_epoch > to_time: + return + + cat_match = (args.cat == '' or current_item.cat.strip() == args.cat) + log_match = (args.log == '' or args.log.lower() in current_item.log.lower()) + logtype_match = (args.logtype is None or current_item.logtype == args.logtype) + + if cat_match and log_match and logtype_match: + result.append(current_item) + +def search(l: Logger, args_: SearchArgs) -> List[LogItem]: + args = args_ + + # Format category (max 10 chars, ascii only) + args.cat = name_fix(args.cat) + if len(args.cat) > 10: + raise ValueError('category cannot be longer than 10 chars') + + timestamp_from = args.timestamp_from if args.timestamp_from else OurTime() + timestamp_to = args.timestamp_to if args.timestamp_to else OurTime() + + # Get time range + from_time = timestamp_from.unix() + to_time = timestamp_to.unix() + if from_time > to_time: + raise ValueError(f'from_time cannot be after to_time: {from_time} < {to_time}') + + result: List[LogItem] = [] + + # Find log files in time range + files = sorted(os.listdir(l.path.path)) + + for file in files: + if not file.endswith('.log'): + continue + + # Parse dayhour from filename + dayhour = file[:-4] # remove .log + try: + file_time = ourtime_new(dayhour) + except ValueError: + continue # Skip if filename is not a valid time format + + current_time = OurTime() + current_item = LogItem(OurTime(), "", "", LogType.STDOUT) # Initialize with dummy values + collecting = False + + # Skip if file is outside time range + if file_time.unix() < from_time or file_time.unix() > to_time: + continue + + # Read and parse log file + content = "" + try: + with open(os.path.join(l.path.path, file), 'r') as f: + content = f.read() + except FileNotFoundError: + continue + + lines = content.split('\n') + + for line in lines: + if len(result) >= args.maxitems: + return result + + line_trim = line.strip() + if not line_trim: + continue + + # Check if this is a timestamp line + if not (line.startswith(' ') or line.startswith('E')): + try: + current_time = ourtime_new(line_trim) + except ValueError: + continue # Skip if not a valid timestamp line + + if collecting: + process(result, current_item, current_time, args, from_time, to_time) + collecting = False + continue + + if collecting and len(line) > 14 and line[13] == '-': + process(result, current_item, current_time, args, from_time, to_time) + collecting = False + + # Parse log line + is_error = line.startswith('E') + if not collecting: + # Start new item + cat_start = 2 + cat_end = 12 + log_start = 15 + + if len(line) < log_start: + continue # Line too short to contain log content + + current_item = LogItem( + timestamp=current_time, + cat=line[cat_start:cat_end].strip(), + log=line[log_start:].strip(), + logtype=LogType.ERROR if is_error else LogType.STDOUT + ) + collecting = True + else: + # Continuation line + if len(line_trim) < 16: # Check for minimum length for continuation line + current_item.log += '\n' + line_trim + else: + current_item.log += '\n' + line[15:].strip() # Use strip for continuation lines + + # Add last item if collecting + if collecting: + process(result, current_item, current_time, args, from_time, to_time) + + return result \ No newline at end of file diff --git a/lib/core/pathlib/__init__.py b/lib/core/pathlib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/core/pathlib/__pycache__/__init__.cpython-313.pyc b/lib/core/pathlib/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..4123a7b Binary files /dev/null and b/lib/core/pathlib/__pycache__/__init__.cpython-313.pyc differ diff --git a/lib/core/pathlib/__pycache__/pathlib.cpython-313.pyc b/lib/core/pathlib/__pycache__/pathlib.cpython-313.pyc new file mode 100644 index 0000000..f5379c9 Binary files /dev/null and b/lib/core/pathlib/__pycache__/pathlib.cpython-313.pyc differ diff --git a/lib/core/pathlib/pathlib.py b/lib/core/pathlib/pathlib.py new file mode 100644 index 0000000..a8e2f27 --- /dev/null +++ b/lib/core/pathlib/pathlib.py @@ -0,0 +1,80 @@ +import os + +class Path: + def __init__(self, path: str): + self.path = os.path.expanduser(path) + + def exists(self) -> bool: + return os.path.exists(self.path) + + def is_file(self) -> bool: + return os.path.isfile(self.path) + + def is_dir(self) -> bool: + return os.path.isdir(self.path) + + def read(self) -> str: + with open(self.path, 'r') as f: + return f.read() + + def write(self, content: str): + os.makedirs(os.path.dirname(self.path), exist_ok=True) + with open(self.path, 'w') as f: + f.write(content) + + def delete(self): + if self.is_file(): + os.remove(self.path) + elif self.is_dir(): + os.rmdir(self.path) + + def list(self, recursive: bool = False, regex: list = None) -> list[str]: + files = [] + if self.is_dir(): + if recursive: + for root, _, filenames in os.walk(self.path): + for filename in filenames: + full_path = os.path.join(root, filename) + relative_path = os.path.relpath(full_path, self.path) + if regex: + import re + if any(re.match(r, relative_path) for r in regex): + files.append(relative_path) + else: + files.append(relative_path) + else: + for entry in os.listdir(self.path): + full_path = os.path.join(self.path, entry) + if os.path.isfile(full_path): + if regex: + import re + if any(re.match(r, entry) for r in regex): + files.append(entry) + else: + files.append(entry) + return files + +def get(path: str) -> Path: + return Path(path) + +def get_dir(path: str, create: bool = False) -> Path: + p = Path(path) + if create and not p.exists(): + os.makedirs(p.path, exist_ok=True) + return p + +def get_file(path: str, create: bool = False) -> Path: + p = Path(path) + if create and not p.exists(): + os.makedirs(os.path.dirname(p.path), exist_ok=True) + with open(p.path, 'w') as f: + pass # Create empty file + return p + +def rmdir_all(path: str): + if os.path.exists(path): + import shutil + shutil.rmtree(path) + +def ls(path: str) -> list[str]: + return os.listdir(path) \ No newline at end of file diff --git a/lib/core/texttools/__init__.py b/lib/core/texttools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/core/texttools/texttools.py b/lib/core/texttools/texttools.py new file mode 100644 index 0000000..5a60056 --- /dev/null +++ b/lib/core/texttools/texttools.py @@ -0,0 +1,142 @@ +import re + +def name_fix(name: str) -> str: + # VLang's name_fix converts '-' to '_' and cleans up special chars. + # Python's re.sub can handle this. + name = re.sub(r'[^a-zA-Z0-9_ ]', '', name.replace('-', '_')) + return name.strip() + +def expand(txt: str, length: int, expand_with: str) -> str: + # Pads the string to the specified length. + return txt.ljust(length, expand_with) + +def dedent(text: str) -> str: + # Removes common leading whitespace from every line. + # This is a simplified version of textwrap.dedent + lines = text.splitlines() + if not lines: + return "" + + # Find the minimum indentation of non-empty lines + min_indent = float('inf') + for line in lines: + if line.strip(): + indent = len(line) - len(line.lstrip()) + min_indent = min(min_indent, indent) + + if min_indent == float('inf'): # All lines are empty or just whitespace + return "\n".join([line.strip() for line in lines]) + + dedented_lines = [line[min_indent:] for line in lines] + return "\n".join(dedented_lines) + +def remove_empty_lines(text: str) -> str: + lines = text.splitlines() + return "\n".join([line for line in lines if line.strip()]) + +def remove_double_lines(text: str) -> str: + lines = text.splitlines() + cleaned_lines = [] + prev_empty = False + for line in lines: + is_empty = not line.strip() + if is_empty and prev_empty: + continue + cleaned_lines.append(line) + prev_empty = is_empty + return "\n".join(cleaned_lines) + +def ascii_clean(r: str) -> str: + return r.encode('ascii', 'ignore').decode('ascii') + +def name_clean(r: str) -> str: + return re.sub(r'[^a-zA-Z0-9]', '', r) + +def name_fix_keepspace(name_: str) -> str: + # Similar to name_fix but keeps spaces. + return re.sub(r'[^a-zA-Z0-9 ]', '', name_.replace('-', '_')).strip() + +def name_fix_no_ext(name_: str) -> str: + return os.path.splitext(name_)[0] + +def name_fix_snake_to_pascal(name: str) -> str: + return ''.join(word.capitalize() for word in name.split('_')) + +def snake_case(name: str) -> str: + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + +def name_split(name: str) -> tuple[str, str]: + parts = name.split('.') + if len(parts) > 1: + return parts[0], '.'.join(parts[1:]) + return name, "" + +def cmd_line_args_parser(text: str) -> list[str]: + # A simple parser, might need more robust solution for complex cases + import shlex + return shlex.split(text) + +def text_remove_quotes(text: str) -> str: + return re.sub(r'["\'].*?["\']', '', text) + +def check_exists_outside_quotes(text: str, items: list[str]) -> bool: + # This is a simplified implementation. A full implementation would require + # more complex parsing to correctly identify text outside quotes. + cleaned_text = text_remove_quotes(text) + for item in items: + if item in cleaned_text: + return True + return False + +def is_int(text: str) -> bool: + return text.isdigit() + +def is_upper_text(text: str) -> bool: + return text.isupper() and text.isalpha() + +def multiline_to_single(text: str) -> str: + return text.replace('\n', '\\n').replace('\r', '') + +def split_smart(t: str, delimiter_: str) -> list[str]: + # This is a placeholder, a smart split would need to handle quotes and escapes + return t.split(delimiter_) + +def version(text_: str) -> int: + # Converts version strings like "v0.4.36" to 4036 or "v1.4.36" to 1004036 + match = re.match(r'v?(\d+)\.(\d+)\.(\d+)', text_) + if match: + major, minor, patch = int(match.group(1)), int(match.group(2)), int(match.group(3)) + if major == 0: + return minor * 100 + patch + else: + return major * 1000000 + minor * 100 + patch + return 0 + +def format_rfc1123(dt: datetime) -> str: + return dt.strftime('%a, %d %b %Y %H:%M:%S GMT') + +def to_array(r: str) -> list[str]: + if ',' in r: + return [item.strip() for item in r.split(',')] + return [item.strip() for item in r.splitlines() if item.strip()] + +def to_array_int(r: str) -> list[int]: + return [int(item) for item in to_array(r) if item.isdigit()] + +def to_map(mapstring: str, line: str, delimiter_: str = ' ') -> dict[str, str]: + # This is a simplified implementation. The VLang version is more complex. + # It assumes a space delimiter for now. + keys = [k.strip() for k in mapstring.split(',')] + values = line.split(delimiter_) + + result = {} + val_idx = 0 + for key in keys: + if key == '-': + val_idx += 1 + continue + if val_idx < len(values): + result[key] = values[val_idx] + val_idx += 1 + return result \ No newline at end of file diff --git a/lib/data/ourtime/__init__.py b/lib/data/ourtime/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/data/ourtime/__pycache__/__init__.cpython-313.pyc b/lib/data/ourtime/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..b15cd97 Binary files /dev/null and b/lib/data/ourtime/__pycache__/__init__.cpython-313.pyc differ diff --git a/lib/data/ourtime/__pycache__/ourtime.cpython-313.pyc b/lib/data/ourtime/__pycache__/ourtime.cpython-313.pyc new file mode 100644 index 0000000..2612319 Binary files /dev/null and b/lib/data/ourtime/__pycache__/ourtime.cpython-313.pyc differ diff --git a/lib/data/ourtime/ourtime.py b/lib/data/ourtime/ourtime.py new file mode 100644 index 0000000..8628359 --- /dev/null +++ b/lib/data/ourtime/ourtime.py @@ -0,0 +1,123 @@ +from datetime import datetime, timedelta +import re + +class OurTime: + def __init__(self, dt: datetime = None): + self._dt = dt if dt else datetime.min + + def __str__(self) -> str: + return self.str() + + def str(self) -> str: + if self._dt == datetime.min: + return "" + return self._dt.strftime('%Y-%m-%d %H:%M') + + def day(self) -> str: + if self._dt == datetime.min: + return "" + return self._dt.strftime('%Y-%m-%d') + + def key(self) -> str: + if self._dt == datetime.min: + return "" + return self._dt.strftime('%Y_%m_%d_%H_%M_%S') + + def md(self) -> str: + if self._dt == datetime.min: + return "" + return self._dt.strftime('%Y-%m-%d %H:%M:%S') + + def unix(self) -> int: + if self._dt == datetime.min: + return 0 + return int(self._dt.timestamp()) + + def empty(self) -> bool: + return self._dt == datetime.min + + def dayhour(self) -> str: + if self._dt == datetime.min: + return "" + return self._dt.strftime('%Y-%m-%d-%H') + + def time(self): + # This is a simplified representation, as VLang's time() returns a time object. + # Here, we return self to allow chaining format_ss(). + return self + + def format_ss(self) -> str: + if self._dt == datetime.min: + return "" + return self._dt.strftime('%H:%M:%S') + + def warp(self, expression: str): + if self._dt == datetime.min: + return + + parts = expression.split() + for part in parts: + match = re.match(r'([+-]?\d+)([smhdwMQY])', part) + if not match: + continue + + value = int(match.group(1)) + unit = match.group(2) + + if unit == 's': + self._dt += timedelta(seconds=value) + elif unit == 'm': + self._dt += timedelta(minutes=value) + elif unit == 'h': + self._dt += timedelta(hours=value) + elif unit == 'd': + self._dt += timedelta(days=value) + elif unit == 'w': + self._dt += timedelta(weeks=value) + elif unit == 'M': + # Approximate months, for more accuracy, a proper dateutil.relativedelta would be needed + self._dt += timedelta(days=value * 30) + elif unit == 'Q': + self._dt += timedelta(days=value * 90) + elif unit == 'Y': + self._dt += timedelta(days=value * 365) + +def now() -> OurTime: + return OurTime(datetime.now()) + +def new(time_str: str) -> OurTime: + if not time_str: + return OurTime() + + formats = [ + '%Y-%m-%d %H:%M:%S', + '%Y-%m-%d %H:%M', + '%Y-%m-%d %H', + '%Y-%m-%d', + '%d-%m-%Y %H:%M:%S', + '%d-%m-%Y %H:%M', + '%d-%m-%Y %H', + '%d-%m-%Y', + '%H:%M:%S', # For time() and format_ss() usage + ] + + for fmt in formats: + try: + dt = datetime.strptime(time_str, fmt) + return OurTime(dt) + except ValueError: + pass + + # Handle relative time expressions + try: + # Create a dummy OurTime object to use its warp method + temp_time = now() + temp_time.warp(time_str) + return temp_time + except Exception: + pass + + raise ValueError(f"Could not parse time string: {time_str}") + +def new_from_epoch(epoch: int) -> OurTime: + return OurTime(datetime.fromtimestamp(epoch)) \ No newline at end of file