Squashed 'components/rfs/' content from commit 9808a5e

git-subtree-dir: components/rfs
git-subtree-split: 9808a5e9fc768edc7d8b1dfa5b91b3f018dff0cb
This commit is contained in:
2025-08-16 21:12:45 +02:00
commit 9790ef4dac
96 changed files with 14003 additions and 0 deletions

52
fl-server/Cargo.toml Normal file
View File

@@ -0,0 +1,52 @@
[package]
name = "fl-server"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[build-dependencies]
git-version = "0.3.5"
[[bin]]
name = "fl-server"
path = "src/main.rs"
[dependencies]
log = "0.4"
anyhow = "1.0.44"
regex = "1.9.6"
rfs = { path = "../rfs"}
docker2fl = { path = "../docker2fl"}
tokio = { version = "1", features = ["full"] }
bollard = "0.15.0"
futures-util = "0.3"
simple_logger = {version = "1.0.1"}
uuid = { version = "1.3.1", features = ["v4"] }
tempdir = "0.3"
serde_json = "1.0"
toml = "0.4.2"
clap = { version = "4.5.8", features = ["derive"] }
serde = { version = "1.0.159" , features = ["derive"] }
axum = "0.7"
axum-macros = "0.4.1"
tower = { version = "0.4", features = ["util", "timeout", "load-shed", "limit"] }
tower-http = { version = "0.5.2", features = ["fs", "cors", "add-extension", "auth", "compression-full", "trace", "limit"] }
tokio-async-drop = "0.1.0"
mime_guess = "2.0.5"
mime = "0.3.17"
percent-encoding = "2.3.1"
tracing = "0.1.40"
askama = "0.12.1"
hyper = { version = "1.4.0", features = ["full"] }
time = { version = "0.3.36", features = ["formatting"] }
chrono = "0.4.38"
jsonwebtoken = "9.3.0"
utoipa = { version = "4", features = ["axum_extras"] }
utoipa-swagger-ui = { version = "7", features = ["axum"] }
thiserror = "1.0.63"
hostname-validator = "1.1.1"
walkdir = "2.5.0"
sha256 = "1.5.0"
async-trait = "0.1.53"

42
fl-server/README.md Normal file
View File

@@ -0,0 +1,42 @@
# Flist server
Flist server helps using rfs and docker2fl tools to generate different flists from docker images.
## Build
```bash
cargo build
```
## Run
First create `config.toml` check [configuration](#configuration)
```bash
cargo run --bin fl-server -- --config-path config.toml -d
```
### Configuration
Before building or running the server, create `config.toml` in the current directory.
example `config.toml`:
```toml
host="Your host to run the server on, required, example: 'localhost'"
port="Your port to run the server on, required, example: 3000, validation: between [0, 65535]"
store_url="List of stores to pack flists in which can be 'dir', 'zdb', 's3', required, example: ['dir:///tmp/store0']"
flist_dir="A directory to save each user flists, required, example: 'flists'"
jwt_secret="secret for jwt, required, example: 'secret'"
jwt_expire_hours="Life time for jwt token in hours, required, example: 5, validation: between [1, 24]"
[[users]] # list of authorized user in the server
username = "user1"
password = "password1"
[[users]]
username = "user2"
password = "password2"
...
```

9
fl-server/build.rs Normal file
View File

@@ -0,0 +1,9 @@
fn main() {
println!(
"cargo:rustc-env=GIT_VERSION={}",
git_version::git_version!(
args = ["--tags", "--always", "--dirty=-modified"],
fallback = "unknown"
)
);
}

154
fl-server/src/auth.rs Normal file
View File

@@ -0,0 +1,154 @@
use std::sync::Arc;
use axum::{
extract::{Json, Request, State},
http::{self, StatusCode},
middleware::Next,
response::IntoResponse,
};
use axum_macros::debug_handler;
use chrono::{Duration, Utc};
use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation};
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use crate::{
config,
response::{ResponseError, ResponseResult},
};
#[derive(Serialize, Deserialize)]
pub struct Claims {
pub exp: usize, // Expiry time of the token
pub iat: usize, // Issued at time of the token
pub username: String, // Username associated with the token
}
#[derive(Deserialize, ToSchema)]
pub struct SignInBody {
pub username: String,
pub password: String,
}
#[derive(Serialize, ToSchema)]
pub struct SignInResponse {
pub access_token: String,
}
#[utoipa::path(
post,
path = "/v1/api/signin",
request_body = SignInBody,
responses(
(status = 200, description = "User signed in successfully", body = SignInResponse),
(status = 500, description = "Internal server error"),
(status = 401, description = "Unauthorized user"),
)
)]
#[debug_handler]
pub async fn sign_in_handler(
State(state): State<Arc<config::AppState>>,
Json(user_data): Json<SignInBody>,
) -> impl IntoResponse {
let user = match state.db.get_user_by_username(&user_data.username) {
Some(user) => user,
None => {
return Err(ResponseError::Unauthorized(
"User is not authorized".to_string(),
));
}
};
if user_data.password != user.password {
return Err(ResponseError::Unauthorized(
"Wrong username or password".to_string(),
));
}
let token = encode_jwt(
user.username.clone(),
state.config.jwt_secret.clone(),
state.config.jwt_expire_hours,
)
.map_err(|_| ResponseError::InternalServerError)?;
Ok(ResponseResult::SignedIn(SignInResponse {
access_token: token,
}))
}
pub fn encode_jwt(
username: String,
jwt_secret: String,
jwt_expire: i64,
) -> Result<String, StatusCode> {
let now = Utc::now();
let exp: usize = (now + Duration::hours(jwt_expire)).timestamp() as usize;
let iat: usize = now.timestamp() as usize;
let claim = Claims { iat, exp, username };
encode(
&Header::default(),
&claim,
&EncodingKey::from_secret(jwt_secret.as_ref()),
)
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
}
pub fn decode_jwt(jwt_token: String, jwt_secret: String) -> Result<TokenData<Claims>, StatusCode> {
let result: Result<TokenData<Claims>, StatusCode> = decode(
&jwt_token,
&DecodingKey::from_secret(jwt_secret.as_ref()),
&Validation::default(),
)
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR);
result
}
pub async fn authorize(
State(state): State<Arc<config::AppState>>,
mut req: Request,
next: Next,
) -> impl IntoResponse {
let auth_header = match req.headers_mut().get(http::header::AUTHORIZATION) {
Some(header) => header
.to_str()
.map_err(|_| ResponseError::Forbidden("Empty header is not allowed".to_string()))?,
None => {
return Err(ResponseError::Forbidden(
"No JWT token is added to the header".to_string(),
))
}
};
let mut header = auth_header.split_whitespace();
let (_, token) = (header.next(), header.next());
let token_str = match token {
Some(t) => t.to_string(),
None => {
log::error!("failed to get token string");
return Err(ResponseError::InternalServerError);
}
};
let token_data = match decode_jwt(token_str, state.config.jwt_secret.clone()) {
Ok(data) => data,
Err(_) => {
return Err(ResponseError::Forbidden(
"Unable to decode JWT token".to_string(),
))
}
};
let current_user = match state.db.get_user_by_username(&token_data.claims.username) {
Some(user) => user,
None => {
return Err(ResponseError::Unauthorized(
"You are not an authorized user".to_string(),
));
}
};
req.extensions_mut().insert(current_user.username.clone());
Ok(next.run(req).await)
}

63
fl-server/src/config.rs Normal file
View File

@@ -0,0 +1,63 @@
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
fs,
path::PathBuf,
sync::{Arc, Mutex},
};
use utoipa::ToSchema;
use crate::{
db::{User, DB},
handlers,
};
#[derive(Debug, ToSchema, Serialize, Clone)]
pub struct Job {
pub id: String,
}
#[derive(ToSchema)]
pub struct AppState {
pub jobs_state: Mutex<HashMap<String, handlers::FlistState>>,
pub flists_progress: Mutex<HashMap<PathBuf, f32>>,
pub db: Arc<dyn DB>,
pub config: Config,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct Config {
pub host: String,
pub port: u16,
pub store_url: Vec<String>,
pub flist_dir: String,
pub jwt_secret: String,
pub jwt_expire_hours: i64,
pub users: Vec<User>,
}
/// Parse the config file into Config struct.
pub async fn parse_config(filepath: &str) -> Result<Config> {
let content = fs::read_to_string(filepath).context("failed to read config file")?;
let c: Config = toml::from_str(&content).context("failed to convert toml config data")?;
if !hostname_validator::is_valid(&c.host) {
anyhow::bail!("host '{}' is invalid", c.host)
}
rfs::store::parse_router(&c.store_url)
.await
.context("failed to parse store urls")?;
fs::create_dir_all(&c.flist_dir).context("failed to create flists directory")?;
if c.jwt_expire_hours < 1 || c.jwt_expire_hours > 24 {
anyhow::bail!(format!(
"jwt expiry interval in hours '{}' is invalid, must be between [1, 24]",
c.jwt_expire_hours
))
}
Ok(c)
}

36
fl-server/src/db.rs Normal file
View File

@@ -0,0 +1,36 @@
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct User {
pub username: String,
pub password: String,
}
pub trait DB: Send + Sync {
fn get_user_by_username(&self, username: &str) -> Option<User>;
}
#[derive(Debug, ToSchema)]
pub struct MapDB {
users: HashMap<String, User>,
}
impl MapDB {
pub fn new(users: &[User]) -> Self {
Self {
users: users
.iter()
.map(|u| (u.username.clone(), u.to_owned()))
.collect(),
}
}
}
impl DB for MapDB {
fn get_user_by_username(&self, username: &str) -> Option<User> {
self.users.get(username).cloned()
}
}

585
fl-server/src/handlers.rs Normal file
View File

@@ -0,0 +1,585 @@
use anyhow::Error;
use axum::{
extract::{Path, Query, State},
response::IntoResponse,
Extension, Json,
};
use axum_macros::debug_handler;
use std::{
collections::HashMap,
fs,
path::PathBuf,
sync::{mpsc, Arc},
};
use bollard::auth::DockerCredentials;
use serde::{Deserialize, Serialize};
use crate::{
auth::{SignInBody, SignInResponse, __path_sign_in_handler},
response::{DirListTemplate, DirLister, ErrorTemplate, TemplateErr},
};
use crate::{
config::{self, Job},
response::{FileInfo, ResponseError, ResponseResult},
serve_flists::visit_dir_one_level,
};
use rfs::fungi::{Reader, Writer};
use utoipa::{OpenApi, ToSchema};
use uuid::Uuid;
#[derive(OpenApi)]
#[openapi(
paths(health_check_handler, create_flist_handler, get_flist_state_handler, preview_flist_handler, list_flists_handler, sign_in_handler),
components(schemas(DirListTemplate, DirLister, FlistBody, Job, ResponseError, ErrorTemplate, TemplateErr, ResponseResult, FileInfo, SignInBody, FlistState, SignInResponse, FlistStateInfo, PreviewResponse)),
tags(
(name = "fl-server", description = "Flist conversion API")
)
)]
pub struct FlistApi;
#[derive(Debug, Deserialize, Serialize, Clone, ToSchema)]
pub struct FlistBody {
#[schema(example = "redis")]
pub image_name: String,
pub username: Option<String>,
pub password: Option<String>,
pub auth: Option<String>,
pub email: Option<String>,
pub server_address: Option<String>,
pub identity_token: Option<String>,
pub registry_token: Option<String>,
}
#[derive(Debug, Deserialize, Serialize, Clone, ToSchema)]
pub struct PreviewResponse {
pub content: Vec<PathBuf>,
pub metadata: String,
pub checksum: String,
}
#[derive(Debug, Clone, Serialize, PartialEq, ToSchema)]
pub enum FlistState {
Accepted(String),
Started(String),
InProgress(FlistStateInfo),
Created(String),
Failed,
}
#[derive(Debug, Clone, Serialize, PartialEq, ToSchema)]
pub struct FlistStateInfo {
msg: String,
progress: f32,
}
const DEFAULT_LIMIT: usize = 10;
const DEFAULT_PAGE: usize = 1;
#[derive(Deserialize)]
pub struct Pagination {
page: Option<usize>,
limit: Option<usize>,
}
#[derive(Deserialize, Clone)]
pub struct Filter {
pub max_size: Option<usize>,
pub min_size: Option<usize>,
username: Option<String>,
pub name: Option<String>,
}
#[utoipa::path(
get,
path = "/v1/api",
responses(
(status = 200, description = "flist server is working", body = String)
)
)]
pub async fn health_check_handler() -> ResponseResult {
ResponseResult::Health
}
#[utoipa::path(
post,
path = "/v1/api/fl",
request_body = FlistBody,
responses(
(status = 201, description = "Flist conversion started", body = Job),
(status = 401, description = "Unauthorized user"),
(status = 403, description = "Forbidden"),
(status = 409, description = "Conflict"),
(status = 500, description = "Internal server error"),
)
)]
#[debug_handler]
pub async fn create_flist_handler(
State(state): State<Arc<config::AppState>>,
Extension(username): Extension<String>,
Json(body): Json<FlistBody>,
) -> impl IntoResponse {
let cfg = state.config.clone();
let credentials = Some(DockerCredentials {
username: body.username,
password: body.password,
auth: body.auth,
email: body.email,
serveraddress: body.server_address,
identitytoken: body.identity_token,
registrytoken: body.registry_token,
});
let mut docker_image = body.image_name.to_string();
if !docker_image.contains(':') {
docker_image.push_str(":latest");
}
let fl_name = docker_image.replace([':', '/'], "-") + ".fl";
let username_dir = std::path::Path::new(&cfg.flist_dir).join(&username);
let fl_path = username_dir.join(&fl_name);
if fl_path.exists() {
return Err(ResponseError::Conflict("flist already exists".to_string()));
}
if let Err(err) = fs::create_dir_all(&username_dir) {
log::error!(
"failed to create user flist directory `{:?}` with error {:?}",
&username_dir,
err
);
return Err(ResponseError::InternalServerError);
}
let meta = match Writer::new(&fl_path, true).await {
Ok(writer) => writer,
Err(err) => {
log::error!(
"failed to create a new writer for flist `{:?}` with error {}",
fl_path,
err
);
return Err(ResponseError::InternalServerError);
}
};
let store = match rfs::store::parse_router(&cfg.store_url).await {
Ok(s) => s,
Err(err) => {
log::error!("failed to parse router for store with error {}", err);
return Err(ResponseError::InternalServerError);
}
};
// Create a new job id for the flist request
let job: Job = Job {
id: Uuid::new_v4().to_string(),
};
let current_job = job.clone();
state
.jobs_state
.lock()
.expect("failed to lock state")
.insert(
job.id.clone(),
FlistState::Accepted(format!("flist '{}' is accepted", &fl_name)),
);
let flist_download_url = std::path::Path::new(&format!("{}:{}", cfg.host, cfg.port))
.join(cfg.flist_dir)
.join(username)
.join(&fl_name);
tokio::spawn(async move {
state
.jobs_state
.lock()
.expect("failed to lock state")
.insert(
job.id.clone(),
FlistState::Started(format!("flist '{}' is started", fl_name)),
);
let container_name = Uuid::new_v4().to_string();
let docker_tmp_dir =
tempdir::TempDir::new(&container_name).expect("failed to create tmp dir for docker");
let (tx, rx) = mpsc::channel();
let mut docker_to_fl =
docker2fl::DockerImageToFlist::new(meta, docker_image, credentials, docker_tmp_dir);
let res = docker_to_fl.prepare().await;
if res.is_err() {
let _ = tokio::fs::remove_file(&fl_path).await;
state
.jobs_state
.lock()
.expect("failed to lock state")
.insert(job.id.clone(), FlistState::Failed);
return;
}
let files_count = docker_to_fl.files_count();
let st = state.clone();
let job_id = job.id.clone();
let cloned_fl_path = fl_path.clone();
tokio::spawn(async move {
let mut progress: f32 = 0.0;
for _ in 0..files_count - 1 {
let step = rx.recv().expect("failed to receive progress") as f32;
progress += step;
let progress_percentage = progress / files_count as f32 * 100.0;
st.jobs_state.lock().expect("failed to lock state").insert(
job_id.clone(),
FlistState::InProgress(FlistStateInfo {
msg: "flist is in progress".to_string(),
progress: progress_percentage,
}),
);
st.flists_progress
.lock()
.expect("failed to lock state")
.insert(cloned_fl_path.clone(), progress_percentage);
}
});
let res = docker_to_fl.pack(store, Some(tx)).await;
// remove the file created with the writer if fl creation failed
if res.is_err() {
log::error!("failed creation failed with error {:?}", res.err());
let _ = tokio::fs::remove_file(&fl_path).await;
state
.jobs_state
.lock()
.expect("failed to lock state")
.insert(job.id.clone(), FlistState::Failed);
return;
}
state
.jobs_state
.lock()
.expect("failed to lock state")
.insert(
job.id.clone(),
FlistState::Created(format!(
"flist {:?} is created successfully",
flist_download_url
)),
);
state
.flists_progress
.lock()
.expect("failed to lock state")
.insert(fl_path, 100.0);
});
Ok(ResponseResult::FlistCreated(current_job))
}
#[utoipa::path(
get,
path = "/v1/api/fl/{job_id}",
responses(
(status = 200, description = "Flist state", body = FlistState),
(status = 404, description = "Flist not found"),
(status = 500, description = "Internal server error"),
(status = 401, description = "Unauthorized user"),
(status = 403, description = "Forbidden"),
),
params(
("job_id" = String, Path, description = "flist job id")
)
)]
#[debug_handler]
pub async fn get_flist_state_handler(
Path(flist_job_id): Path<String>,
State(state): State<Arc<config::AppState>>,
) -> impl IntoResponse {
if !&state
.jobs_state
.lock()
.expect("failed to lock state")
.contains_key(&flist_job_id.clone())
{
return Err(ResponseError::NotFound("flist doesn't exist".to_string()));
}
let res_state = state
.jobs_state
.lock()
.expect("failed to lock state")
.get(&flist_job_id.clone())
.expect("failed to get from state")
.to_owned();
match res_state {
FlistState::Accepted(_) => Ok(ResponseResult::FlistState(res_state)),
FlistState::Started(_) => Ok(ResponseResult::FlistState(res_state)),
FlistState::InProgress(_) => Ok(ResponseResult::FlistState(res_state)),
FlistState::Created(_) => {
state
.jobs_state
.lock()
.expect("failed to lock state")
.remove(&flist_job_id.clone());
Ok(ResponseResult::FlistState(res_state))
}
FlistState::Failed => {
state
.jobs_state
.lock()
.expect("failed to lock state")
.remove(&flist_job_id.clone());
Err(ResponseError::InternalServerError)
}
}
}
#[utoipa::path(
get,
path = "/v1/api/fl",
responses(
(status = 200, description = "Listing flists", body = HashMap<String, Vec<FileInfo>>),
(status = 401, description = "Unauthorized user"),
(status = 403, description = "Forbidden"),
(status = 500, description = "Internal server error"),
)
)]
#[debug_handler]
pub async fn list_flists_handler(
State(state): State<Arc<config::AppState>>,
pagination: Query<Pagination>,
filter: Query<Filter>,
) -> impl IntoResponse {
let mut flists: HashMap<String, Vec<FileInfo>> = HashMap::new();
let pagination: Pagination = pagination.0;
let page = pagination.page.unwrap_or(DEFAULT_PAGE);
let limit = pagination.limit.unwrap_or(DEFAULT_LIMIT);
if page == 0 {
return Err(ResponseError::BadRequest(
"requested page should be nonzero positive number".to_string(),
));
}
let filter: Filter = filter.0;
let rs: Result<Vec<FileInfo>, std::io::Error> =
visit_dir_one_level(&state.config.flist_dir, &state, None).await;
let files = match rs {
Ok(files) => files,
Err(e) => {
log::error!("failed to list flists directory with error: {}", e);
return Err(ResponseError::InternalServerError);
}
};
for file in files {
if !file.is_file {
let flists_per_username =
visit_dir_one_level(&file.path_uri, &state, Some(filter.clone())).await;
if let Some(ref filter_username) = filter.username {
if filter_username.clone() != file.name {
continue;
}
}
match flists_per_username {
Ok(files) => {
let username = file.name;
flists.insert(username.clone(), Vec::new());
let start = limit * (page - 1);
let end = limit * page;
if files.len() > start {
if files.len() >= end {
flists.insert(username, files[start..end].to_vec());
} else {
flists.insert(username, files[start..].to_vec());
}
}
}
Err(e) => {
log::error!("failed to list flists per username with error: {}", e);
return Err(ResponseError::InternalServerError);
}
};
};
}
Ok(ResponseResult::Flists(flists))
}
#[utoipa::path(
get,
path = "/v1/api/fl/preview/{flist_path}",
responses(
(status = 200, description = "Flist preview result", body = PreviewResponse),
(status = 400, description = "Bad request"),
(status = 401, description = "Unauthorized user"),
(status = 403, description = "Forbidden"),
(status = 500, description = "Internal server error"),
),
params(
("flist_path" = String, Path, description = "flist file path")
)
)]
#[debug_handler]
pub async fn preview_flist_handler(
State(state): State<Arc<config::AppState>>,
Path(flist_path): Path<String>,
) -> impl IntoResponse {
let fl_path = flist_path;
match validate_flist_path(&state, &fl_path).await {
Ok(_) => (),
Err(err) => return Err(ResponseError::BadRequest(err.to_string())),
};
let content = match get_flist_content(&fl_path).await {
Ok(paths) => paths,
Err(_) => return Err(ResponseError::InternalServerError),
};
let bytes = match std::fs::read(&fl_path) {
Ok(b) => b,
Err(err) => {
log::error!(
"failed to read flist '{}' into bytes with error {}",
fl_path,
err
);
return Err(ResponseError::InternalServerError);
}
};
Ok(ResponseResult::PreviewFlist(PreviewResponse {
content,
metadata: state.config.store_url.join("-"),
checksum: sha256::digest(&bytes),
}))
}
async fn validate_flist_path(state: &Arc<config::AppState>, fl_path: &String) -> Result<(), Error> {
// validate path starting with `/`
if fl_path.starts_with("/") {
anyhow::bail!("invalid flist path '{}', shouldn't start with '/'", fl_path);
}
// path should include 3 parts [parent dir, username, flist file]
let parts: Vec<_> = fl_path.split("/").collect();
if parts.len() != 3 {
anyhow::bail!(
format!("invalid flist path '{}', should consist of 3 parts [parent directory, username and flist name", fl_path
));
}
// validate parent dir
if parts[0] != state.config.flist_dir {
anyhow::bail!(
"invalid flist path '{}', parent directory should be '{}'",
fl_path,
state.config.flist_dir
);
}
// validate username
match state.db.get_user_by_username(&parts[1]) {
Some(_) => (),
None => {
anyhow::bail!(
"invalid flist path '{}', username '{}' doesn't exist",
fl_path,
parts[1]
);
}
};
// validate flist extension
let fl_name = parts[2].to_string();
let ext = match std::path::Path::new(&fl_name).extension() {
Some(ex) => ex.to_string_lossy().to_string(),
None => "".to_string(),
};
if ext != "fl" {
anyhow::bail!(
"invalid flist path '{}', invalid flist extension '{}' should be 'fl'",
fl_path,
ext
);
}
// validate flist existence
if !std::path::Path::new(parts[0])
.join(parts[1])
.join(&fl_name)
.exists()
{
anyhow::bail!("flist '{}' doesn't exist", fl_path);
}
Ok(())
}
async fn get_flist_content(fl_path: &String) -> Result<Vec<PathBuf>, Error> {
let mut visitor = ReadVisitor::default();
let meta = match Reader::new(&fl_path).await {
Ok(reader) => reader,
Err(err) => {
log::error!(
"failed to initialize metadata database for flist `{}` with error {}",
fl_path,
err
);
anyhow::bail!("Internal server error");
}
};
match meta.walk(&mut visitor).await {
Ok(()) => return Ok(visitor.into_inner()),
Err(err) => {
log::error!(
"failed to walk through metadata for flist `{}` with error {}",
fl_path,
err
);
anyhow::bail!("Internal server error");
}
};
}
#[derive(Default)]
struct ReadVisitor {
inner: Vec<PathBuf>,
}
impl ReadVisitor {
pub fn into_inner(self) -> Vec<PathBuf> {
self.inner
}
}
#[async_trait::async_trait]
impl rfs::fungi::meta::WalkVisitor for ReadVisitor {
async fn visit(
&mut self,
path: &std::path::Path,
_node: &rfs::fungi::meta::Inode,
) -> rfs::fungi::meta::Result<rfs::fungi::meta::Walk> {
self.inner.push(path.to_path_buf());
Ok(rfs::fungi::meta::Walk::Continue)
}
}

186
fl-server/src/main.rs Normal file
View File

@@ -0,0 +1,186 @@
mod auth;
mod config;
mod db;
mod handlers;
mod response;
mod serve_flists;
use anyhow::{Context, Result};
use axum::{
error_handling::HandleErrorLayer,
http::StatusCode,
middleware,
response::IntoResponse,
routing::{get, post},
BoxError, Router,
};
use clap::{ArgAction, Parser};
use hyper::{
header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE},
Method,
};
use std::{
borrow::Cow,
collections::HashMap,
sync::{Arc, Mutex},
time::Duration,
};
use tokio::{runtime::Builder, signal};
use tower::ServiceBuilder;
use tower_http::cors::CorsLayer;
use tower_http::{cors::Any, trace::TraceLayer};
use utoipa::OpenApi;
use utoipa_swagger_ui::SwaggerUi;
#[derive(Parser, Debug)]
#[clap(name ="fl-server", author, version = env!("GIT_VERSION"), about, long_about = None)]
struct Options {
/// enable debugging logs
#[clap(short, long, action=ArgAction::Count)]
debug: u8,
/// config file path
#[clap(short, long)]
config_path: String,
}
fn main() -> Result<()> {
let rt = Builder::new_multi_thread()
.thread_stack_size(8 * 1024 * 1024)
.enable_all()
.build()
.unwrap();
rt.block_on(app())
}
async fn app() -> Result<()> {
let opts = Options::parse();
simple_logger::SimpleLogger::new()
.with_utc_timestamps()
.with_level({
match opts.debug {
0 => log::LevelFilter::Info,
1 => log::LevelFilter::Debug,
_ => log::LevelFilter::Trace,
}
})
.with_module_level("sqlx", log::Level::Error.to_level_filter())
.init()?;
let config = config::parse_config(&opts.config_path)
.await
.context("failed to parse config file")?;
let db = Arc::new(db::MapDB::new(&config.users.clone()));
let app_state = Arc::new(config::AppState {
jobs_state: Mutex::new(HashMap::new()),
flists_progress: Mutex::new(HashMap::new()),
db,
config,
});
let cors = CorsLayer::new()
.allow_origin(Any)
.allow_methods([Method::GET, Method::POST])
.allow_headers([AUTHORIZATION, ACCEPT, CONTENT_TYPE]);
let v1_routes = Router::new()
.route("/v1/api", get(handlers::health_check_handler))
.route("/v1/api/signin", post(auth::sign_in_handler))
.route(
"/v1/api/fl",
post(handlers::create_flist_handler).layer(middleware::from_fn_with_state(
app_state.clone(),
auth::authorize,
)),
)
.route(
"/v1/api/fl/:job_id",
get(handlers::get_flist_state_handler).layer(middleware::from_fn_with_state(
app_state.clone(),
auth::authorize,
)),
)
.route(
"/v1/api/fl/preview/:flist_path",
get(handlers::preview_flist_handler),
)
.route("/v1/api/fl", get(handlers::list_flists_handler))
.route("/*path", get(serve_flists::serve_flists));
let app = Router::new()
.merge(
SwaggerUi::new("/swagger-ui")
.url("/api-docs/openapi.json", handlers::FlistApi::openapi()),
)
.merge(v1_routes)
.layer(
ServiceBuilder::new()
.layer(HandleErrorLayer::new(handle_error))
.load_shed()
.concurrency_limit(1024)
.timeout(Duration::from_secs(10))
.layer(TraceLayer::new_for_http()),
)
.with_state(Arc::clone(&app_state))
.layer(cors);
let address = format!("{}:{}", app_state.config.host, app_state.config.port);
let listener = tokio::net::TcpListener::bind(address)
.await
.context("failed to bind address")?;
log::info!(
"🚀 Server started successfully at {}:{}",
app_state.config.host,
app_state.config.port
);
axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal())
.await
.context("failed to serve listener")?;
Ok(())
}
async fn shutdown_signal() {
let ctrl_c = async {
signal::ctrl_c()
.await
.expect("failed to install Ctrl+C handler");
};
#[cfg(unix)]
let terminate = async {
signal::unix::signal(signal::unix::SignalKind::terminate())
.expect("failed to install signal handler")
.recv()
.await;
};
tokio::select! {
_ = ctrl_c => {},
_ = terminate => {},
}
}
async fn handle_error(error: BoxError) -> impl IntoResponse {
if error.is::<tower::timeout::error::Elapsed>() {
return (StatusCode::REQUEST_TIMEOUT, Cow::from("request timed out"));
}
if error.is::<tower::load_shed::error::Overloaded>() {
return (
StatusCode::SERVICE_UNAVAILABLE,
Cow::from("service is overloaded, try again later"),
);
}
(
StatusCode::INTERNAL_SERVER_ERROR,
Cow::from(format!("Unhandled internal error: {}", error)),
)
}

178
fl-server/src/response.rs Normal file
View File

@@ -0,0 +1,178 @@
use std::collections::HashMap;
use askama::Template;
use axum::{
body::Body,
http::StatusCode,
response::{Html, IntoResponse, Response},
Json,
};
use serde::Serialize;
use utoipa::ToSchema;
use crate::{
auth::SignInResponse,
config::Job,
handlers::{FlistState, PreviewResponse},
};
#[derive(Serialize, ToSchema)]
pub enum ResponseError {
InternalServerError,
Conflict(String),
NotFound(String),
Unauthorized(String),
BadRequest(String),
Forbidden(String),
TemplateError(ErrorTemplate),
}
impl IntoResponse for ResponseError {
fn into_response(self) -> Response<Body> {
match self {
ResponseError::InternalServerError => {
(StatusCode::INTERNAL_SERVER_ERROR, "Internal server error").into_response()
}
ResponseError::Conflict(msg) => (StatusCode::CONFLICT, msg).into_response(),
ResponseError::NotFound(msg) => (StatusCode::NOT_FOUND, msg).into_response(),
ResponseError::Unauthorized(msg) => (StatusCode::UNAUTHORIZED, msg).into_response(),
ResponseError::BadRequest(msg) => (StatusCode::BAD_REQUEST, msg).into_response(),
ResponseError::Forbidden(msg) => (StatusCode::FORBIDDEN, msg).into_response(),
ResponseError::TemplateError(t) => match t.render() {
Ok(html) => {
let mut resp = Html(html).into_response();
match t.err {
TemplateErr::NotFound(reason) => {
*resp.status_mut() = StatusCode::NOT_FOUND;
resp.headers_mut()
.insert(FAIL_REASON_HEADER_NAME, reason.parse().unwrap());
}
TemplateErr::BadRequest(reason) => {
*resp.status_mut() = StatusCode::BAD_REQUEST;
resp.headers_mut()
.insert(FAIL_REASON_HEADER_NAME, reason.parse().unwrap());
}
TemplateErr::InternalServerError(reason) => {
*resp.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
resp.headers_mut()
.insert(FAIL_REASON_HEADER_NAME, reason.parse().unwrap());
}
}
resp
}
Err(err) => {
tracing::error!("template render failed, err={}", err);
(
StatusCode::INTERNAL_SERVER_ERROR,
format!("Failed to render template. Error: {}", err),
)
.into_response()
}
},
}
}
}
#[derive(ToSchema)]
pub enum ResponseResult {
Health,
FlistCreated(Job),
FlistState(FlistState),
Flists(HashMap<String, Vec<FileInfo>>),
PreviewFlist(PreviewResponse),
SignedIn(SignInResponse),
DirTemplate(DirListTemplate),
Res(hyper::Response<tower_http::services::fs::ServeFileSystemResponseBody>),
}
impl IntoResponse for ResponseResult {
fn into_response(self) -> Response<Body> {
match self {
ResponseResult::Health => (
StatusCode::OK,
Json(serde_json::json!({"msg": "flist server is working"})),
)
.into_response(),
ResponseResult::SignedIn(token) => (StatusCode::CREATED, Json(token)).into_response(),
ResponseResult::FlistCreated(job) => (StatusCode::CREATED, Json(job)).into_response(),
ResponseResult::FlistState(flist_state) => (
StatusCode::OK,
Json(serde_json::json!({
"flist_state": flist_state
})),
)
.into_response(),
ResponseResult::Flists(flists) => (StatusCode::OK, Json(flists)).into_response(),
ResponseResult::PreviewFlist(content) => {
(StatusCode::OK, Json(content)).into_response()
}
ResponseResult::DirTemplate(t) => match t.render() {
Ok(html) => Html(html).into_response(),
Err(err) => {
tracing::error!("template render failed, err={}", err);
(
StatusCode::INTERNAL_SERVER_ERROR,
format!("Failed to render template. Error: {}", err),
)
.into_response()
}
},
ResponseResult::Res(res) => res.map(axum::body::Body::new),
}
}
}
//////// TEMPLATES ////////
#[derive(Serialize, Clone, Debug, ToSchema)]
pub struct FileInfo {
pub name: String,
pub path_uri: String,
pub is_file: bool,
pub size: u64,
pub last_modified: i64,
pub progress: f32,
}
#[derive(Serialize, ToSchema)]
pub struct DirLister {
pub files: Vec<FileInfo>,
}
#[derive(Template, Serialize, ToSchema)]
#[template(path = "index.html")]
pub struct DirListTemplate {
pub lister: DirLister,
pub cur_path: String,
}
mod filters {
pub(crate) fn datetime(ts: &i64) -> ::askama::Result<String> {
if let Ok(format) =
time::format_description::parse("[year]-[month]-[day] [hour]:[minute]:[second] UTC")
{
return Ok(time::OffsetDateTime::from_unix_timestamp(*ts)
.unwrap()
.format(&format)
.unwrap());
}
Err(askama::Error::Fmt(std::fmt::Error))
}
}
#[derive(Template, Serialize, ToSchema)]
#[template(path = "error.html")]
pub struct ErrorTemplate {
pub err: TemplateErr,
pub cur_path: String,
pub message: String,
}
const FAIL_REASON_HEADER_NAME: &str = "fl-server-fail-reason";
#[derive(Serialize, ToSchema)]
pub enum TemplateErr {
BadRequest(String),
NotFound(String),
InternalServerError(String),
}

View File

@@ -0,0 +1,174 @@
use axum::extract::State;
use std::{io::Error, path::PathBuf, sync::Arc};
use tokio::io;
use tower::util::ServiceExt;
use tower_http::services::ServeDir;
use axum::{
body::Body,
http::{Request, StatusCode},
response::IntoResponse,
};
use axum_macros::debug_handler;
use percent_encoding::percent_decode;
use crate::{
config,
handlers::Filter,
response::{
DirListTemplate, DirLister, ErrorTemplate, FileInfo, ResponseError, ResponseResult,
TemplateErr,
},
};
#[debug_handler]
pub async fn serve_flists(
State(state): State<Arc<config::AppState>>,
req: Request<Body>,
) -> impl IntoResponse {
let path = req.uri().path().to_string();
return match ServeDir::new("").oneshot(req).await {
Ok(res) => {
let status = res.status();
match status {
StatusCode::NOT_FOUND => {
let full_path = match validate_path(&path) {
Ok(p) => p,
Err(_) => {
return Err(ResponseError::TemplateError(ErrorTemplate {
err: TemplateErr::BadRequest("invalid path".to_string()),
cur_path: path.to_string(),
message: "invalid path".to_owned(),
}));
}
};
let cur_path = std::path::Path::new(&full_path);
match cur_path.is_dir() {
true => {
let rs = visit_dir_one_level(&full_path, &state, None).await;
match rs {
Ok(files) => Ok(ResponseResult::DirTemplate(DirListTemplate {
lister: DirLister { files },
cur_path: path.to_string(),
})),
Err(e) => Err(ResponseError::TemplateError(ErrorTemplate {
err: TemplateErr::InternalServerError(e.to_string()),
cur_path: path.to_string(),
message: e.to_string(),
})),
}
}
false => Err(ResponseError::TemplateError(ErrorTemplate {
err: TemplateErr::NotFound("file not found".to_string()),
cur_path: path.to_string(),
message: "file not found".to_owned(),
})),
}
}
_ => Ok(ResponseResult::Res(res)),
}
}
Err(err) => Err(ResponseError::TemplateError(ErrorTemplate {
err: TemplateErr::InternalServerError(format!("Unhandled error: {}", err)),
cur_path: path.to_string(),
message: format!("Unhandled error: {}", err),
})),
};
}
fn validate_path(path: &str) -> io::Result<PathBuf> {
let path = path.trim_start_matches('/');
let path = percent_decode(path.as_ref()).decode_utf8_lossy();
let mut full_path = PathBuf::new();
// validate
for seg in path.split('/') {
if seg.starts_with("..") || seg.contains('\\') {
return Err(Error::other("invalid path"));
}
full_path.push(seg);
}
Ok(full_path)
}
pub async fn visit_dir_one_level<P: AsRef<std::path::Path>>(
path: P,
state: &Arc<config::AppState>,
filter: Option<Filter>,
) -> io::Result<Vec<FileInfo>> {
let path = path.as_ref();
let mut dir = tokio::fs::read_dir(path).await?;
let mut files: Vec<FileInfo> = Vec::new();
while let Some(child) = dir.next_entry().await? {
let path_uri = child.path().to_string_lossy().to_string();
let is_file = child.file_type().await?.is_file();
let name = child.file_name().to_string_lossy().to_string();
let size = child.metadata().await?.len();
let mut progress = 0.0;
if is_file {
match state
.flists_progress
.lock()
.expect("failed to lock state")
.get(&path.join(&name).to_path_buf())
{
Some(p) => progress = *p,
None => progress = 100.0,
}
let ext = child
.path()
.extension()
.expect("failed to get path extension")
.to_string_lossy()
.to_string();
if ext != "fl" {
continue;
}
}
if let Some(ref filter_files) = filter {
if let Some(ref filter_name) = filter_files.name {
if filter_name.clone() != name {
continue;
}
}
if let Some(ref filter_max_size) = filter_files.max_size {
if filter_max_size.clone() < size as usize {
continue;
}
}
if let Some(ref filter_min_size) = filter_files.min_size {
if filter_min_size.clone() > size as usize {
continue;
}
}
}
files.push(FileInfo {
name,
path_uri,
is_file,
size: size,
last_modified: child
.metadata()
.await?
.modified()?
.duration_since(std::time::SystemTime::UNIX_EPOCH)
.expect("failed to get duration")
.as_secs() as i64,
progress,
});
}
Ok(files)
}

View File

@@ -0,0 +1,57 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>{% block title %}{{ title }}{% endblock %}</title>
<style>
a {
text-decoration: none;
color: #4c7883;
cursor: pointer;
}
a:hover, a:hover .directory, a:hover .file {
color: #6ea90c;
}
.directory {
color:#6ea90c;
}
.file {
color: #585a56;
}
#main {
width: 60%;
max-width: 1600px;
margin: 1em auto;
}
footer {
display: block;
padding: 40px;
font-size: 12px;
text-align: center;
}
</style>
{% block head %}{% endblock %}
</head>
<body>
<div id="main">
{% block content %}{% endblock %}
</div>
<footer>
Served with <a href="https://github.com/threefoldtech/rfs">flist-server</a>
</footer>
{% block foot %}{% endblock %}
</body>
</html>

View File

@@ -0,0 +1,16 @@
{% extends "base.html" %}
{% block title %}Error Occured when listing directory for {{ cur_path }}{% endblock %}
{% block content %}
<h1>Error</h1>
<hr>
<dl>
<dt> Request Path: </dt> <dd> <span class="directory">{{ cur_path }}</span> </dd>
<dt> Error Message: </dt> <dd> <span class="error">{{ message }} </span> </dd>
</dl>
<hr />
{% endblock %}

View File

@@ -0,0 +1,41 @@
{% extends "base.html" %}
{% block title %}Directory listing for /{{ cur_path }}{% endblock %}
{% block head %}
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
{% endblock %}
{% block content %}
<h1>Directory listing for <span class="directory">/{{ cur_path }}</span></h1>
<hr>
<ol>
{% for file in lister.files %}
{% if file.is_file %}
<li class="item">
<a class="file"
href="/{{ file.path_uri|urlencode }}"
data-src="{{ file.path_uri|urlencode }}"
title="{{file.name}} {{ file.last_modified|datetime }}">
<span class="fa fa-file"></span> {{file.name}}
</a>
</li>
{% else %}
<li>
<a href="/{{ file.path_uri|urlencode }}/" title="{{ file.last_modified|datetime }}">
<span class="fa fa-folder"></span> {{file.name}}/
</a>
</li>
{% endif %}
{% endfor %}
</ol>
<hr>
{% endblock %}