This commit is contained in:
kristof 2025-04-02 08:24:03 +02:00
parent 3606e27e30
commit b2896b206c
11 changed files with 739 additions and 15 deletions

View File

@ -15,6 +15,13 @@ readme = "README.md"
libc = "0.2"
cfg-if = "1.0"
thiserror = "1.0" # For error handling
redis = "0.22.0" # Redis client
lazy_static = "1.4.0" # For lazy initialization of static variables
regex = "1.8.1" # For regex pattern matching
serde = { version = "1.0", features = ["derive"] } # For serialization/deserialization
serde_json = "1.0" # For JSON handling
glob = "0.3.1" # For file pattern matching
tempfile = "3.5" # For temporary file operations
log = "0.4" # Logging facade
# Optional features for specific OS functionality

1
src/env/mod.rs vendored Normal file
View File

@ -0,0 +1 @@
pub mod redisclient;

178
src/env/redisclient.rs vendored Normal file
View File

@ -0,0 +1,178 @@
use redis::{Client, Connection, Commands, RedisError, RedisResult, Cmd};
use std::env;
use std::path::Path;
use std::sync::{Arc, Mutex, Once};
use std::sync::atomic::{AtomicBool, Ordering};
use lazy_static::lazy_static;
// Global Redis client instance using lazy_static
lazy_static! {
static ref REDIS_CLIENT: Mutex<Option<Arc<RedisClientWrapper>>> = Mutex::new(None);
static ref INIT: Once = Once::new();
}
// Wrapper for Redis client to handle connection and DB selection
pub struct RedisClientWrapper {
client: Client,
connection: Mutex<Option<Connection>>,
db: i64,
initialized: AtomicBool,
}
impl RedisClientWrapper {
// Create a new Redis client wrapper
fn new(client: Client, db: i64) -> Self {
RedisClientWrapper {
client,
connection: Mutex::new(None),
db,
initialized: AtomicBool::new(false),
}
}
// Execute a command on the Redis connection
pub fn execute<T: redis::FromRedisValue>(&self, cmd: &mut Cmd) -> RedisResult<T> {
let mut conn_guard = self.connection.lock().unwrap();
// If we don't have a connection or it's not working, create a new one
if conn_guard.is_none() || {
if let Some(ref mut conn) = *conn_guard {
let ping_result: RedisResult<String> = redis::cmd("PING").query(conn);
ping_result.is_err()
} else {
true
}
} {
*conn_guard = Some(self.client.get_connection()?);
}
cmd.query(&mut conn_guard.as_mut().unwrap())
}
// Initialize the client (ping and select DB)
fn initialize(&self) -> RedisResult<()> {
if self.initialized.load(Ordering::Relaxed) {
return Ok(());
}
let mut conn = self.client.get_connection()?;
// Ping Redis to ensure it works
let ping_result: String = redis::cmd("PING").query(&mut conn)?;
if ping_result != "PONG" {
return Err(RedisError::from((redis::ErrorKind::ResponseError, "Failed to ping Redis server")));
}
// Select the database
redis::cmd("SELECT").arg(self.db).execute(&mut conn);
self.initialized.store(true, Ordering::Relaxed);
// Store the connection
let mut conn_guard = self.connection.lock().unwrap();
*conn_guard = Some(conn);
Ok(())
}
}
// Get the Redis client instance
pub fn get_redis_client() -> RedisResult<Arc<RedisClientWrapper>> {
// Check if we already have a client
{
let guard = REDIS_CLIENT.lock().unwrap();
if let Some(ref client) = &*guard {
return Ok(Arc::clone(client));
}
}
// Create a new client
let client = create_redis_client()?;
// Store the client globally
{
let mut guard = REDIS_CLIENT.lock().unwrap();
*guard = Some(Arc::clone(&client));
}
Ok(client)
}
// Create a new Redis client
fn create_redis_client() -> RedisResult<Arc<RedisClientWrapper>> {
// First try: Connect via Unix socket
let home_dir = env::var("HOME").unwrap_or_else(|_| String::from("/root"));
let socket_path = format!("{}/hero/var/myredis.sock", home_dir);
if Path::new(&socket_path).exists() {
// Try to connect via Unix socket
let socket_url = format!("unix://{}", socket_path);
match Client::open(socket_url) {
Ok(client) => {
let db = get_redis_db();
let wrapper = Arc::new(RedisClientWrapper::new(client, db));
// Initialize the client
if let Err(err) = wrapper.initialize() {
eprintln!("Socket exists at {} but connection failed: {}", socket_path, err);
} else {
return Ok(wrapper);
}
},
Err(err) => {
eprintln!("Socket exists at {} but connection failed: {}", socket_path, err);
}
}
}
// Second try: Connect via TCP to localhost
let tcp_url = "redis://127.0.0.1/";
match Client::open(tcp_url) {
Ok(client) => {
let db = get_redis_db();
let wrapper = Arc::new(RedisClientWrapper::new(client, db));
// Initialize the client
wrapper.initialize()?;
Ok(wrapper)
},
Err(err) => {
Err(RedisError::from((
redis::ErrorKind::IoError,
"Failed to connect to Redis",
format!("Could not connect via socket at {} or via TCP to localhost: {}", socket_path, err)
)))
}
}
}
// Get the Redis DB number from environment variable
fn get_redis_db() -> i64 {
env::var("REDISDB")
.ok()
.and_then(|db_str| db_str.parse::<i64>().ok())
.unwrap_or(0)
}
// Reload the Redis client
pub fn reset() -> RedisResult<()> {
// Clear the existing client
{
let mut client_guard = REDIS_CLIENT.lock().unwrap();
*client_guard = None;
}
// Create a new client, only return error if it fails
// We don't need to return the client itself
get_redis_client()?;
Ok(())
}
// Execute a Redis command
pub fn execute<T>(cmd: &mut Cmd) -> RedisResult<T>
where
T: redis::FromRedisValue,
{
let client = get_redis_client()?;
client.execute(cmd)
}

View File

@ -1,7 +1,355 @@
use std::process::Command;
use std::path::Path;
use std::fs;
use std::env;
use regex::Regex;
use std::fmt;
use std::process::{Command, Output};
use std::error::Error;
use std::fmt;
use std::collections::HashMap;
use redis::Cmd;
use serde::{Deserialize, Serialize};
use crate::env::redisclient;
use crate::git::git::parse_git_url;
// Define a custom error type for GitExecutor operations
#[derive(Debug)]
pub enum GitExecutorError {
GitCommandFailed(String),
CommandExecutionError(std::io::Error),
RedisError(redis::RedisError),
JsonError(serde_json::Error),
AuthenticationError(String),
SshAgentNotLoaded,
InvalidAuthConfig(String),
}
// Implement Display for GitExecutorError
impl fmt::Display for GitExecutorError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
GitExecutorError::GitCommandFailed(e) => write!(f, "Git command failed: {}", e),
GitExecutorError::CommandExecutionError(e) => write!(f, "Command execution error: {}", e),
GitExecutorError::RedisError(e) => write!(f, "Redis error: {}", e),
GitExecutorError::JsonError(e) => write!(f, "JSON error: {}", e),
GitExecutorError::AuthenticationError(e) => write!(f, "Authentication error: {}", e),
GitExecutorError::SshAgentNotLoaded => write!(f, "SSH agent is not loaded"),
GitExecutorError::InvalidAuthConfig(e) => write!(f, "Invalid authentication configuration: {}", e),
}
}
}
// Implement Error trait for GitExecutorError
impl Error for GitExecutorError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
GitExecutorError::CommandExecutionError(e) => Some(e),
GitExecutorError::RedisError(e) => Some(e),
GitExecutorError::JsonError(e) => Some(e),
_ => None,
}
}
}
// From implementations for error conversion
impl From<redis::RedisError> for GitExecutorError {
fn from(err: redis::RedisError) -> Self {
GitExecutorError::RedisError(err)
}
}
impl From<serde_json::Error> for GitExecutorError {
fn from(err: serde_json::Error) -> Self {
GitExecutorError::JsonError(err)
}
}
impl From<std::io::Error> for GitExecutorError {
fn from(err: std::io::Error) -> Self {
GitExecutorError::CommandExecutionError(err)
}
}
// Status enum for GitConfig
#[derive(Debug, Serialize, Deserialize, PartialEq)]
pub enum GitConfigStatus {
#[serde(rename = "error")]
Error,
#[serde(rename = "ok")]
Ok,
}
// Auth configuration for a specific git server
#[derive(Debug, Serialize, Deserialize)]
pub struct GitServerAuth {
pub sshagent: Option<bool>,
pub key: Option<String>,
pub username: Option<String>,
pub password: Option<String>,
}
// Main configuration structure from Redis
#[derive(Debug, Serialize, Deserialize)]
pub struct GitConfig {
pub status: GitConfigStatus,
pub auth: HashMap<String, GitServerAuth>,
}
// GitExecutor struct
pub struct GitExecutor {
config: Option<GitConfig>,
}
impl GitExecutor {
// Create a new GitExecutor
pub fn new() -> Self {
GitExecutor { config: None }
}
// Initialize by loading configuration from Redis
pub fn init(&mut self) -> Result<(), GitExecutorError> {
// Try to load config from Redis
match self.load_config_from_redis() {
Ok(config) => {
self.config = Some(config);
Ok(())
}
Err(e) => {
// If Redis error, we'll proceed without config
// This is not a fatal error as we might use default git behavior
eprintln!("Warning: Failed to load git config from Redis: {}", e);
self.config = None;
Ok(())
}
}
}
// Load configuration from Redis
fn load_config_from_redis(&self) -> Result<GitConfig, GitExecutorError> {
// Create Redis command to get the herocontext:git key
let mut cmd = Cmd::new();
cmd.arg("GET").arg("herocontext:git");
// Execute the command
let result: redis::RedisResult<String> = redisclient::execute(&mut cmd);
match result {
Ok(json_str) => {
// Parse the JSON string into GitConfig
let config: GitConfig = serde_json::from_str(&json_str)?;
// Validate the config
if config.status == GitConfigStatus::Error {
return Err(GitExecutorError::InvalidAuthConfig("Config status is error".to_string()));
}
Ok(config)
}
Err(e) => Err(GitExecutorError::RedisError(e)),
}
}
// Check if SSH agent is loaded
fn is_ssh_agent_loaded(&self) -> bool {
let output = Command::new("ssh-add")
.arg("-l")
.output();
match output {
Ok(output) => output.status.success() && !output.stdout.is_empty(),
Err(_) => false,
}
}
// Get authentication configuration for a git URL
fn get_auth_for_url(&self, url: &str) -> Option<&GitServerAuth> {
if let Some(config) = &self.config {
let (server, _, _) = parse_git_url(url);
if !server.is_empty() {
return config.auth.get(&server);
}
}
None
}
// Validate authentication configuration
fn validate_auth_config(&self, auth: &GitServerAuth) -> Result<(), GitExecutorError> {
// Rule: If sshagent is true, other fields should be empty
if let Some(true) = auth.sshagent {
if auth.key.is_some() || auth.username.is_some() || auth.password.is_some() {
return Err(GitExecutorError::InvalidAuthConfig(
"When sshagent is true, key, username, and password must be empty".to_string()
));
}
// Check if SSH agent is actually loaded
if !self.is_ssh_agent_loaded() {
return Err(GitExecutorError::SshAgentNotLoaded);
}
}
// Rule: If key is set, other fields should be empty
if let Some(_) = &auth.key {
if auth.sshagent.unwrap_or(false) || auth.username.is_some() || auth.password.is_some() {
return Err(GitExecutorError::InvalidAuthConfig(
"When key is set, sshagent, username, and password must be empty".to_string()
));
}
}
// Rule: If username is set, password should be set and other fields empty
if let Some(_) = &auth.username {
if auth.sshagent.unwrap_or(false) || auth.key.is_some() {
return Err(GitExecutorError::InvalidAuthConfig(
"When username is set, sshagent and key must be empty".to_string()
));
}
if auth.password.is_none() {
return Err(GitExecutorError::InvalidAuthConfig(
"When username is set, password must also be set".to_string()
));
}
}
Ok(())
}
// Execute a git command with authentication
pub fn execute(&self, args: &[&str]) -> Result<Output, GitExecutorError> {
// Extract the git URL if this is a command that needs authentication
let url_arg = self.extract_git_url_from_args(args);
// If we have a URL and authentication config, use it
if let Some(url) = url_arg {
if let Some(auth) = self.get_auth_for_url(&url) {
// Validate the authentication configuration
self.validate_auth_config(auth)?;
// Execute with the appropriate authentication method
return self.execute_with_auth(args, auth);
}
}
// No special authentication needed, execute normally
self.execute_git_command(args)
}
// Extract git URL from command arguments
fn extract_git_url_from_args<'a>(&self, args: &[&'a str]) -> Option<&'a str> {
// Commands that might contain a git URL
if args.contains(&"clone") || args.contains(&"fetch") || args.contains(&"pull") || args.contains(&"push") {
// The URL is typically the last argument for clone, or after remote for others
for (i, &arg) in args.iter().enumerate() {
if arg == "clone" && i + 1 < args.len() {
return Some(args[i + 1]);
}
if (arg == "fetch" || arg == "pull" || arg == "push") && i + 1 < args.len() {
// For these commands, the URL might be specified as a remote name
// We'd need more complex logic to resolve remote names to URLs
// For now, we'll just return None
return None;
}
}
}
None
}
// Execute git command with authentication
fn execute_with_auth(&self, args: &[&str], auth: &GitServerAuth) -> Result<Output, GitExecutorError> {
// Handle different authentication methods
if let Some(true) = auth.sshagent {
// Use SSH agent (already validated that it's loaded)
self.execute_git_command(args)
} else if let Some(key) = &auth.key {
// Use SSH key
self.execute_with_ssh_key(args, key)
} else if let Some(username) = &auth.username {
// Use username/password
if let Some(password) = &auth.password {
self.execute_with_credentials(args, username, password)
} else {
// This should never happen due to validation
Err(GitExecutorError::AuthenticationError("Password is required when username is set".to_string()))
}
} else {
// No authentication method specified, use default
self.execute_git_command(args)
}
}
// Execute git command with SSH key
fn execute_with_ssh_key(&self, args: &[&str], key: &str) -> Result<Output, GitExecutorError> {
// Create a command with GIT_SSH_COMMAND to specify the key
let ssh_command = format!("ssh -i {} -o IdentitiesOnly=yes", key);
let mut command = Command::new("git");
command.env("GIT_SSH_COMMAND", ssh_command);
command.args(args);
let output = command.output()?;
if output.status.success() {
Ok(output)
} else {
let error = String::from_utf8_lossy(&output.stderr);
Err(GitExecutorError::GitCommandFailed(error.to_string()))
}
}
// Execute git command with username/password
fn execute_with_credentials(&self, args: &[&str], username: &str, password: &str) -> Result<Output, GitExecutorError> {
// Helper method to execute a command and handle the result
fn execute_command(command: &mut Command) -> Result<Output, GitExecutorError> {
let output = command.output()?;
if output.status.success() {
Ok(output)
} else {
let error = String::from_utf8_lossy(&output.stderr);
Err(GitExecutorError::GitCommandFailed(error.to_string()))
}
}
// For HTTPS authentication, we need to modify the URL to include credentials
// Create a new vector to hold our modified arguments
let modified_args: Vec<String> = args.iter().map(|&arg| {
if arg.starts_with("https://") {
// Replace https:// with https://username:password@
format!("https://{}:{}@{}",
username,
password,
&arg[8..]) // Skip the "https://" part
} else {
arg.to_string()
}
}).collect();
// Execute the command
let mut command = Command::new("git");
// Add the modified arguments to the command
for arg in &modified_args {
command.arg(arg.as_str());
}
// Execute the command and handle the result
let output = command.output()?;
if output.status.success() { Ok(output) } else { Err(GitExecutorError::GitCommandFailed(String::from_utf8_lossy(&output.stderr).to_string())) }
}
// Basic git command execution
fn execute_git_command(&self, args: &[&str]) -> Result<Output, GitExecutorError> {
let mut command = Command::new("git");
command.args(args);
let output = command.output()?;
if output.status.success() {
Ok(output)
} else {
let error = String::from_utf8_lossy(&output.stderr);
Err(GitExecutorError::GitCommandFailed(error.to_string()))
}
}
}
// Implement Default for GitExecutor
impl Default for GitExecutor {
fn default() -> Self {
Self::new()
}
}

View File

@ -1,12 +1,32 @@
in @/sal/git/git_executor.rs
create a GitExecutor which is the one executing git commands
and also checking if ssh-agent is loaded
or if there is an authentication mechanism defined in redis
check if there is a redis on ~/hero/var/redis.sock
how is this done use src/env/redisclient.rs
this allows us to execute redis commands
over unix domani socket
check there is herocontext:git in the redis
if yes then check there is an entry on
if yes fetch the object, its a json representing a struct
sal::git::
- status (error, ok) as enum
- auth which is a map
- key is the server part of our git url (see parse_git_url in git module)
- val is another object with following properties
- sshagent as bool (means if set just use loaded sshagent)
- key (is the sshkey as needs to be used when talking to the server)
- username (if username then there needs to be a password)
- password
we need to deserialize this struct
this now tells based on the server name how to authenticate for the git server
if sshagent then rest needs to be empty
if key rest needs to be empty
if username then password set, rest empty
the git executor needs to use above to talk in right way to the server

View File

@ -1,2 +1,6 @@
use git::*;
use git_executor::*;
mod git;
mod git_executor;
pub use git::*;
pub use git_executor::*;

View File

@ -39,7 +39,6 @@ pub type Result<T> = std::result::Result<T, Error>;
pub mod process;
pub mod git;
pub mod os;
pub mod network;
pub mod env;
pub mod text;

View File

@ -1,3 +1,4 @@
pub mod os;
pub mod env;
pub mod text;
pub mod git;

View File

@ -1,2 +1,5 @@
use fs::*;
use download::*;
mod fs;
mod download;
pub use fs::*;
pub use download::*;

114
src/text/README.md Normal file
View File

@ -0,0 +1,114 @@
# Text Processing Utilities
A collection of Rust utilities for common text processing operations.
## Overview
This module provides functions for text manipulation tasks such as:
- Removing indentation from multiline strings
- Adding prefixes to multiline strings
- Normalizing filenames and paths
## Functions
### Text Indentation
#### `dedent(text: &str) -> String`
Removes common leading whitespace from multiline strings.
```rust
let indented = " line 1\n line 2\n line 3";
let dedented = dedent(indented);
assert_eq!(dedented, "line 1\nline 2\n line 3");
```
**Features:**
- Analyzes all non-empty lines to determine minimum indentation
- Preserves empty lines but removes all leading whitespace from them
- Treats tabs as 4 spaces for indentation purposes
#### `prefix(text: &str, prefix: &str) -> String`
Adds a specified prefix to each line of a multiline string.
```rust
let text = "line 1\nline 2\nline 3";
let prefixed = prefix(text, " ");
assert_eq!(prefixed, " line 1\n line 2\n line 3");
```
### Filename and Path Normalization
#### `name_fix(text: &str) -> String`
Normalizes filenames by:
- Converting to lowercase
- Replacing whitespace and special characters with underscores
- Removing non-ASCII characters
- Collapsing consecutive special characters into a single underscore
```rust
assert_eq!(name_fix("Hello World"), "hello_world");
assert_eq!(name_fix("File-Name.txt"), "file_name.txt");
assert_eq!(name_fix("Résumé"), "rsum");
```
#### `path_fix(text: &str) -> String`
Applies `name_fix()` to the filename portion of a path while preserving the directory structure.
```rust
assert_eq!(path_fix("/path/to/File Name.txt"), "/path/to/file_name.txt");
assert_eq!(path_fix("./relative/path/to/DOCUMENT-123.pdf"), "./relative/path/to/document_123.pdf");
```
**Features:**
- Preserves paths ending with `/` (directories)
- Only normalizes the filename portion, leaving the path structure intact
- Handles both absolute and relative paths
## Usage
Import the functions from the module:
```rust
use your_crate::text::{dedent, prefix, name_fix, path_fix};
```
## Examples
### Cleaning up indented text from a template
```rust
let template = "
<div>
<h1>Title</h1>
<p>
Some paragraph text
with multiple lines
</p>
</div>
";
let clean = dedent(template);
// Result:
// <div>
// <h1>Title</h1>
// <p>
// Some paragraph text
// with multiple lines
// </p>
// </div>
```
### Normalizing user-provided filenames
```rust
let user_filename = "My Document (2023).pdf";
let safe_filename = name_fix(user_filename);
// Result: "my_document_2023_.pdf"
let user_path = "/uploads/User Files/Report #123.xlsx";
let safe_path = path_fix(user_path);
// Result: "/uploads/User Files/report_123.xlsx"

View File

@ -81,3 +81,52 @@ pub fn dedent(text: &str) -> String {
.collect::<Vec<String>>()
.join("\n")
}
/**
* Prefix a multiline string with a specified prefix.
*
* This function adds the specified prefix to the beginning of each line in the input text.
*
* # Arguments
*
* * `text` - The multiline string to prefix
* * `prefix` - The prefix to add to each line
*
* # Returns
*
* * `String` - The prefixed string
*
* # Examples
*
* ```
* let text = "line 1\nline 2\nline 3";
* let prefixed = prefix(text, " ");
* assert_eq!(prefixed, " line 1\n line 2\n line 3");
* ```
*/
pub fn prefix(text: &str, prefix: &str) -> String {
text.lines()
.map(|line| format!("{}{}", prefix, line))
.collect::<Vec<String>>()
.join("\n")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_dedent() {
let indented = " line 1\n line 2\n line 3";
let dedented = dedent(indented);
assert_eq!(dedented, "line 1\nline 2\n line 3");
}
#[test]
fn test_prefix() {
let text = "line 1\nline 2\nline 3";
let prefixed = prefix(text, " ");
assert_eq!(prefixed, " line 1\n line 2\n line 3");
}
}