...

README.md

@@ -5,7 +5,7 @@ has some usefull stuff as well
 
 ## Installation
 
-You can install `herolib` directly from the Git repository using `uv pip` (or `pip`):
+You can install `herolib` directly from the Git repository using `uv pip`:
 
 ```bash
 uv pip install git+https://git.ourworld.tf/herocode/herolib_python.git
@@ -27,6 +27,10 @@ import herolib.core.loghandler.mylogging
 from herolib.core.loghandler.mylogging import MyLogger
 ```
 
+## how to integrate python in other projects
+
+see [Python Herolib Integration](pythonsetup/README.md)
+
 ## Version Control
 
 This library follows standard Git version control practices. Releases can be managed by tagging specific commits in the Git repository. Users installing directly from the Git URL can specify a particular branch, tag, or commit hash to get a specific version. For example:

herolib/core/logger/log_test.py

@@ -1,10 +1,11 @@
 import unittest
 import os
 import shutil
-from lib.core.logger.factory import new
-from lib.core.logger.model import LogItemArgs, LogType, Logger # Import Logger class
-from lib.data.ourtime.ourtime import new as ourtime_new, now as ourtime_now
-from lib.core.pathlib.pathlib import get_file, ls, rmdir_all
+from herolib.core.logger.factory import new
+from herolib.core.logger.model import LogItemArgs, LogType, Logger # Import Logger class
+from herolib.data.ourtime.ourtime import new as ourtime_new, now as ourtime_now
+from herolib.core.pathlib.pathlib import get_file, ls, rmdir_all
+from herolib.core.logger.search import search, SearchArgs
 
 class TestLogger(unittest.TestCase):
     def setUp(self):
@@ -85,18 +86,18 @@ class TestLogger(unittest.TestCase):
         self.assertEqual(len(files), 2) # Expecting two files: 2022-12-05-20.log and 2022-12-05-22.log
 
         # Test search functionality
-        items_stdout = logger.search(
+        items_stdout = search(logger, SearchArgs(
             timestamp_from=ourtime_new('2022-11-01 20:14:35'),
             timestamp_to=ourtime_new('2025-11-01 20:14:35'),
             logtype=LogType.STDOUT
-        )
+        ))
         self.assertEqual(len(items_stdout), 2)
 
-        items_error = logger.search(
+        items_error = search(logger, SearchArgs(
             timestamp_from=ourtime_new('2022-11-01 20:14:35'),
             timestamp_to=ourtime_new('2025-11-01 20:14:35'),
             logtype=LogType.ERROR
-        )
+        ))
         self.assertEqual(len(items_error), 4)
 
         # Test specific log content
@@ -115,34 +116,34 @@ class TestLogger(unittest.TestCase):
         self.assertTrue(found_stdout_log, "Expected stdout log content not found")
 
         # Test search by category
-        items_test_app = logger.search(
+        items_test_app = search(logger, SearchArgs(
             timestamp_from=ourtime_new('2022-11-01 20:14:35'),
             timestamp_to=ourtime_new('2025-11-01 20:14:35'),
             cat='test-app'
-        )
+        ))
         self.assertEqual(len(items_test_app), 2)
 
-        items_error_test = logger.search(
+        items_error_test = search(logger, SearchArgs(
             timestamp_from=ourtime_new('2022-11-01 20:14:35'),
             timestamp_to=ourtime_new('2025-11-01 20:14:35'),
             cat='error-test'
-        )
+        ))
         self.assertEqual(len(items_error_test), 4)
 
         # Test search by log content
-        items_with_aaa = logger.search(
+        items_with_aaa = search(logger, SearchArgs(
             timestamp_from=ourtime_new('2022-11-01 20:14:35'),
             timestamp_to=ourtime_new('2025-11-01 20:14:35'),
             log='aaa'
-        )
+        ))
         self.assertEqual(len(items_with_aaa), 2)
 
         # Test search with timestamp range
-        items_specific_time = logger.search(
+        items_specific_time = search(logger, SearchArgs(
             timestamp_from=ourtime_new('2022-12-05 22:00:00'),
             timestamp_to=ourtime_new('2022-12-05 23:00:00'),
             logtype=LogType.ERROR
-        )
+        ))
         self.assertEqual(len(items_specific_time), 2)
 
 

herolib/core/logger/search.py

@@ -16,122 +16,98 @@ class SearchArgs:
         self.logtype = logtype
         self.maxitems = maxitems
 
-def process(result: List[LogItem], current_item: LogItem, current_time: OurTime,
-            args: SearchArgs, from_time: int, to_time: int):
-    # Add previous item if it matches filters
-    log_epoch = current_item.timestamp.unix()
-    if log_epoch < from_time or log_epoch > to_time:
-        return
-
-    cat_match = (args.cat == '' or current_item.cat.strip() == args.cat)
-    log_match = (args.log == '' or args.log.lower() in current_item.log.lower())
-    logtype_match = (args.logtype is None or current_item.logtype == args.logtype)
-
-    if cat_match and log_match and logtype_match:
-        result.append(current_item)
-
 def search(l: Logger, args_: SearchArgs) -> List[LogItem]:
     args = args_
 
-    # Format category (max 10 chars, ascii only)
     args.cat = name_fix(args.cat)
     if len(args.cat) > 10:
         raise ValueError('category cannot be longer than 10 chars')
 
-    timestamp_from = args.timestamp_from if args.timestamp_from else OurTime()
-    timestamp_to = args.timestamp_to if args.timestamp_to else OurTime()
+    from_time = args.timestamp_from.unix() if args.timestamp_from else 0
+    to_time = args.timestamp_to.unix() if args.timestamp_to else ourtime_new('2100-01-01').unix()
 
-    # Get time range
-    from_time = timestamp_from.unix()
-    to_time = timestamp_to.unix()
     if from_time > to_time:
-        raise ValueError(f'from_time cannot be after to_time: {from_time} < {to_time}')
-
+        raise ValueError(f'from_time cannot be after to_time: {from_time} > {to_time}')
+    
     result: List[LogItem] = []
-
-    # Find log files in time range
+    if not os.path.exists(l.path.path):
+        return []
     files = sorted(os.listdir(l.path.path))
 
     for file in files:
         if not file.endswith('.log'):
             continue
 
-        # Parse dayhour from filename
-        dayhour = file[:-4] # remove .log
+        dayhour = file[:-4]
         try:
             file_time = ourtime_new(dayhour)
         except ValueError:
-            continue # Skip if filename is not a valid time format
-
-        current_time = OurTime()
-        current_item = LogItem(OurTime(), "", "", LogType.STDOUT) # Initialize with dummy values
-        collecting = False
-
-        # Skip if file is outside time range
-        if file_time.unix() < from_time or file_time.unix() > to_time:
             continue
 
-        # Read and parse log file
-        content = ""
+        file_hour_start_unix = file_time.unix()
+        file_hour_end_unix = file_hour_start_unix + 3599
+        if file_hour_end_unix < from_time or file_hour_start_unix > to_time:
+            continue
+        
         try:
             with open(os.path.join(l.path.path, file), 'r') as f:
                 content = f.read()
         except FileNotFoundError:
             continue
 
-        lines = content.split('\n')
+        current_time = None
+        current_item = None
 
-        for line in lines:
+        for line in content.splitlines():
             if len(result) >= args.maxitems:
-                return result
+                break
 
-            line_trim = line.strip()
-            if not line_trim:
+            if not line.strip() and current_item:
+                if from_time <= current_item.timestamp.unix() <= to_time:
+                    if (not args.cat or args.cat == current_item.cat) and \
+                       (not args.log or args.log.lower() in current_item.log.lower()) and \
+                       (args.logtype is None or args.logtype == current_item.logtype):
+                        result.append(current_item)
+                current_item = None
                 continue
 
-            # Check if this is a timestamp line
-            if not (line.startswith(' ') or line.startswith('E')):
+            if not line.startswith(' ') and not line.startswith('E'):
+                if current_item:
+                    if from_time <= current_item.timestamp.unix() <= to_time:
+                         if (not args.cat or args.cat == current_item.cat) and \
+                            (not args.log or args.log.lower() in current_item.log.lower()) and \
+                            (args.logtype is None or args.logtype == current_item.logtype):
+                             result.append(current_item)
                 try:
-                    current_time = ourtime_new(line_trim)
+                    current_time = ourtime_new(f"{file_time.day()} {line.strip()}")
+                    current_item = None
                 except ValueError:
-                    continue # Skip if not a valid timestamp line
+                    current_time = None
+                    current_item = None
+            elif current_time:
+                if line.startswith(' ') or line.startswith('E'):
+                    if len(line) > 14 and line[13] == '-':
+                        if current_item:
+                           if from_time <= current_item.timestamp.unix() <= to_time:
+                                if (not args.cat or args.cat == current_item.cat) and \
+                                   (not args.log or args.log.lower() in current_item.log.lower()) and \
+                                   (args.logtype is None or args.logtype == current_item.logtype):
+                                    result.append(current_item)
+                        
+                        is_error = line.startswith('E')
+                        logtype = LogType.ERROR if is_error else LogType.STDOUT
+                        cat = line[2:12].strip()
+                        log_content = line[15:]
 
-                if collecting:
-                    process(result, current_item, current_time, args, from_time, to_time)
-                collecting = False
-                continue
-
-            if collecting and len(line) > 14 and line[13] == '-':
-                process(result, current_item, current_time, args, from_time, to_time)
-                collecting = False
-
-            # Parse log line
-            is_error = line.startswith('E')
-            if not collecting:
-                # Start new item
-                cat_start = 2
-                cat_end = 12
-                log_start = 15
-
-                if len(line) < log_start:
-                    continue # Line too short to contain log content
-
-                current_item = LogItem(
-                    timestamp=current_time,
-                    cat=line[cat_start:cat_end].strip(),
-                    log=line[log_start:].strip(),
-                    logtype=LogType.ERROR if is_error else LogType.STDOUT
-                )
-                collecting = True
-            else:
-                # Continuation line
-                if len(line_trim) < 16: # Check for minimum length for continuation line
-                    current_item.log += '\n' + line_trim
-                else:
-                    current_item.log += '\n' + line[15:].strip() # Use strip for continuation lines
-
-        # Add last item if collecting
-        if collecting:
-            process(result, current_item, current_time, args, from_time, to_time)
+                        current_item = LogItem(timestamp=current_time, cat=cat, log=log_content.strip(), logtype=logtype)
+                    elif current_item:
+                        current_item.log += "\n" + (line[15:] if len(line) >15 else line)
+        
+        if current_item:
+            if from_time <= current_item.timestamp.unix() <= to_time:
+                if (not args.cat or args.cat == current_item.cat) and \
+                   (not args.log or args.log.lower() in current_item.log.lower()) and \
+                   (args.logtype is None or args.logtype == current_item.logtype):
+                    result.append(current_item)
 
     return result

herolib/core/texttools/texttools.py

@@ -1,4 +1,6 @@
 import re
+from datetime import datetime
+import os
 
 def name_fix(name: str) -> str:
     # VLang's name_fix converts '-' to '_' and cleans up special chars.

pythonsetup/README.md

@@ -0,0 +1,60 @@
+# Python Example setup
+
+This repository contains the setup scripts and configuration files for how we use python in threefold/ourworld
+
+```bash
+./install.sh
+```
+
+This script will:
+
+1. Check if `uv` is installed (a fast Python package installer and resolver)
+2. Install `uv` if it's not found
+3. Initialize a uv project if `pyproject.toml` doesn't exist
+4. Sync all project dependencies using `uv sync`
+5. Install the herolib package from git repository
+6. Create necessary directories: (remove if you don't need this for your project)
+   - `static/css`, `static/js`, `static/images` for static assets
+   - `templates` for HTML templates
+   - `md` for markdown files
+
+check how we install herolib here
+   - `pip install git+https://github.com/ThreeFoldTech/herolib.git`  and also have support for the local checked out version
+
+## Running the Server
+
+### Production Mode
+
+To start the server in production mode, run:
+
+```bash
+./start_server.sh
+```
+
+This script will:
+1. Set environment variables for production
+2. Check and free port 9922 if it's in use
+3. Start the web server on port 9922
+4. Make the server available at http://localhost:9922
+
+### Development/Debug Mode
+
+To start the server in development/debug mode, run:
+
+```bash
+./start_server_debug.sh
+```
+
+This script will:
+1. Set environment variables for development
+2. Enable debug mode
+3. Check and free port 9922 if it's in use
+4. Start the web server on port 9922 with debug options
+5. Make the server available at http://localhost:9922
+
+## Environment Setup
+
+The `pipenv.sh` script is automatically sourced by the startup scripts and handles:
+1. Setting the PYTHONPATH to include the src directory
+2. Creating a virtual environment with uv if it doesn't exist
+3. Activating the virtual environment

pythonsetup/install.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# KnowledgeCenter Web Server Installation Script
+# This script sets up the necessary environment for the Flask web server.
+
+set -e  # Exit on any error
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+echo -e "${BLUE}🔧 Setting up KnowledgeCenter Web Server Environment${NC}"
+echo "=================================================="
+
+# Check if uv is installed
+if ! command -v uv &> /dev/null; then
+    echo -e "${YELLOW}⚠️  uv is not installed. Installing uv...${NC}"
+    curl -LsSf https://astral.sh/uv/install.sh | sh
+    source $HOME/.cargo/env
+    echo -e "${GREEN}✅ uv installed${NC}"
+fi
+
+echo -e "${GREEN}✅ uv found${NC}"
+
+# Initialize uv project if not already done
+if [ ! -f "pyproject.toml" ]; then
+    echo -e "${YELLOW}⚠️  No pyproject.toml found. Initializing uv project...${NC}"
+    uv init --no-readme --python 3.13
+    echo -e "${GREEN}✅ uv project initialized${NC}"
+fi
+
+# Sync dependencies
+echo -e "${YELLOW}📦 Installing dependencies with uv...${NC}"
+uv sync
+if [ -d "$HOME/code/git.ourworld.tf/herocode/herolib_python/herolib" ]; then
+    echo -e "${GREEN}✅ Found local herolib, installing...${NC}"
+    uv pip install -e "$HOME/code/git.ourworld.tf/herocode/herolib_python"
+else
+    echo -e "${YELLOW}📦 Local herolib not found, installing from git...${NC}"
+    uv pip install herolib@git+https://git.ourworld.tf/herocode/herolib_python.git --force-reinstall --no-cache-dir
+fi
+echo -e "${GREEN}✅ Dependencies installed${NC}"
+
+# Create necessary directories
+mkdir -p static/css static/js static/images
+mkdir -p templates
+mkdir -p md
+
+echo -e "${GREEN}✅ Directory structure verified${NC}"
+
+echo -e "${GREEN}🎉 Installation complete! You can now run start_server.sh${NC}"

pythonsetup/pipenv.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+export PYTHONPATH=$PYTHONPATH:$(pwd)/src
+
+
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+echo "Setting up KnowledgeCenter environment in: $SCRIPT_DIR"
+
+# Create virtual environment if it doesn't exist
+if [ ! -d ".venv" ]; then
+    echo "📦 Creating Python virtual environment..."
+    uv venv
+    echo "✅ Virtual environment created"
+else
+    echo "✅ Virtual environment already exists"
+fi
+
+# Activate virtual environment
+echo "🔄 Activating virtual environment..."
+source .venv/bin/activate

pythonsetup/pyproject.toml

@@ -0,0 +1,23 @@
+[project]
+name = "KnowledgeCenter"
+version = "0.1.0"
+description = "Add your description here"
+requires-python = ">=3.13"
+dependencies = [
+    "beautifulsoup4>=4.13.4",
+    "flask>=2.3.3",
+    "markdown>=3.5.1",
+    "Werkzeug>=3.1.3",
+    "peewee>=3.17.0",
+    "pygments>=2.16.1",
+    "toml",
+    "flask_socketio",
+    "eventlet",
+    "fastapi>=0.104.0",
+    "uvicorn>=0.24.0",
+    "python-multipart>=0.0.6",
+    "requests>=2.31.0",
+    "herolib @ git+https://git.ourworld.tf/herocode/herolib_python.git",
+    "pudb",
+    "ipython"
+]

pythonsetup/start_server.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# KnowledgeCenter Web Server Startup Script
+# This script starts the Flask web server on port 9922 for PRODUCTION
+
+set -e  # Exit on any error
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+source pipenv.sh
+
+echo -e "${BLUE}🚀 KnowledgeCenter Web Server Startup (PRODUCTION)${NC}"
+echo "================================================="
+
+# Check if uv is installed
+# Check if port 9922 is available
+if lsof -Pi :9922 -sTCP:LISTEN -t >/dev/null 2>&1; then
+    echo -e "${YELLOW}⚠️  Port 9922 is already in use. Attempting to stop existing process...${NC}"
+    PID=$(lsof -ti:9922)
+    if [ ! -z "$PID" ]; then
+        kill -9 $PID 2>/dev/null || true
+        sleep 2
+        echo -e "${GREEN}✅ Existing process stopped${NC}"
+    fi
+fi
+
+# Set environment variables for production
+export FLASK_APP=src/app.py
+export FLASK_ENV=production
+export FLASK_DEBUG=0
+
+# Display startup information
+echo ""
+echo -e "${BLUE}📋 Server Information:${NC}"
+echo "  • Application: KnowledgeCenter Interest Registration"
+echo "  • Port: 9922"
+echo "  • URL: http://localhost:9922"
+echo "  • Environment: Production"
+echo "  • Debug Mode: Disabled"
+echo ""
+
+# Function to handle cleanup on exit
+cleanup() {
+    echo -e "\n${YELLOW}🛑 Shutting down server...${NC}"
+    exit 0
+}
+
+# Set trap for cleanup
+trap cleanup SIGINT SIGTERM
+
+# Start the Flask development server
+echo -e "${GREEN}🌟 Starting KnowledgeCenter web server...${NC}"
+echo -e "${BLUE}📡 Server will be available at: http://localhost:9922${NC}"
+echo -e "${YELLOW}💡 Press Ctrl+C to stop the server${NC}"
+echo ""
+
+# Start the server with uv, specifying production config
+uv run python src/app.py --cfg prod --port 9922 --host 0.0.0.0
+
+# This line should not be reached unless the server exits
+echo -e "${RED}❌ Server stopped unexpectedly${NC}"

pythonsetup/start_server_debug.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# KnowledgeCenter Web Server Startup Script
+# This script starts the Flask web server on port 9922 for PRODUCTION
+
+set -e  # Exit on any error
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+source pipenv.sh
+
+echo -e "${BLUE}🚀 KnowledgeCenter Web Server Startup (DEVELOPMENT/DEBUG)${NC}"
+echo "================================================="
+
+# Check if uv is installed
+# Check if port 9922 is available
+if lsof -Pi :9922 -sTCP:LISTEN -t >/dev/null 2>&1; then
+    echo -e "${YELLOW}⚠️  Port 9922 is already in use. Attempting to stop existing process...${NC}"
+    PID=$(lsof -ti:9922)
+    if [ ! -z "$PID" ]; then
+        kill -9 $PID 2>/dev/null || true
+        sleep 2
+        echo -e "${GREEN}✅ Existing process stopped${NC}"
+    fi
+fi
+
+# Set environment variables for production
+export FLASK_APP=src/app.py
+export FLASK_ENV=development
+export FLASK_DEBUG=1
+
+# Display startup information
+echo ""
+echo -e "${BLUE}📋 Server Information:${NC}"
+echo "  • Application: KnowledgeCenter Interest Registration"
+echo "  • Port: 9922"
+echo "  • URL: http://localhost:9922"
+echo "  • Environment: Development"
+echo "  • Debug Mode: Enabled"
+echo ""
+
+# Function to handle cleanup on exit
+cleanup() {
+    echo -e "\n${YELLOW}🛑 Shutting down server...${NC}"
+    exit 0
+}
+
+# Set trap for cleanup
+trap cleanup SIGINT SIGTERM
+
+# Start the Flask development server
+echo -e "${GREEN}🌟 Starting KnowledgeCenter web server...${NC}"
+echo -e "${BLUE}📡 Server will be available at: http://localhost:9922${NC}"
+echo -e "${YELLOW}💡 Press Ctrl+C to stop the server${NC}"
+echo ""
+
+# Start the server with uv, specifying production config
+uv run python src/app.py --cfg dev --port 9922 --host 0.0.0.0 --debug
+
+# This line should not be reached unless the server exits
+echo -e "${RED}❌ Server stopped unexpectedly${NC}"