Merge commit '9790ef4dacdf729d8825dbe745379bd6c669b9dd' as 'components/rfs'
This commit is contained in:
52
components/rfs/fl-server/Cargo.toml
Normal file
52
components/rfs/fl-server/Cargo.toml
Normal file
@@ -0,0 +1,52 @@
|
||||
[package]
|
||||
name = "fl-server"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
[build-dependencies]
|
||||
git-version = "0.3.5"
|
||||
|
||||
[[bin]]
|
||||
name = "fl-server"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
log = "0.4"
|
||||
anyhow = "1.0.44"
|
||||
regex = "1.9.6"
|
||||
rfs = { path = "../rfs"}
|
||||
docker2fl = { path = "../docker2fl"}
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
bollard = "0.15.0"
|
||||
futures-util = "0.3"
|
||||
simple_logger = {version = "1.0.1"}
|
||||
uuid = { version = "1.3.1", features = ["v4"] }
|
||||
tempdir = "0.3"
|
||||
serde_json = "1.0"
|
||||
toml = "0.4.2"
|
||||
clap = { version = "4.5.8", features = ["derive"] }
|
||||
|
||||
serde = { version = "1.0.159" , features = ["derive"] }
|
||||
axum = "0.7"
|
||||
axum-macros = "0.4.1"
|
||||
tower = { version = "0.4", features = ["util", "timeout", "load-shed", "limit"] }
|
||||
tower-http = { version = "0.5.2", features = ["fs", "cors", "add-extension", "auth", "compression-full", "trace", "limit"] }
|
||||
tokio-async-drop = "0.1.0"
|
||||
mime_guess = "2.0.5"
|
||||
mime = "0.3.17"
|
||||
percent-encoding = "2.3.1"
|
||||
tracing = "0.1.40"
|
||||
askama = "0.12.1"
|
||||
hyper = { version = "1.4.0", features = ["full"] }
|
||||
time = { version = "0.3.36", features = ["formatting"] }
|
||||
chrono = "0.4.38"
|
||||
jsonwebtoken = "9.3.0"
|
||||
|
||||
utoipa = { version = "4", features = ["axum_extras"] }
|
||||
utoipa-swagger-ui = { version = "7", features = ["axum"] }
|
||||
thiserror = "1.0.63"
|
||||
hostname-validator = "1.1.1"
|
||||
walkdir = "2.5.0"
|
||||
sha256 = "1.5.0"
|
||||
async-trait = "0.1.53"
|
||||
42
components/rfs/fl-server/README.md
Normal file
42
components/rfs/fl-server/README.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# Flist server
|
||||
|
||||
Flist server helps using rfs and docker2fl tools to generate different flists from docker images.
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
## Run
|
||||
|
||||
First create `config.toml` check [configuration](#configuration)
|
||||
|
||||
```bash
|
||||
cargo run --bin fl-server -- --config-path config.toml -d
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
Before building or running the server, create `config.toml` in the current directory.
|
||||
|
||||
example `config.toml`:
|
||||
|
||||
```toml
|
||||
host="Your host to run the server on, required, example: 'localhost'"
|
||||
port="Your port to run the server on, required, example: 3000, validation: between [0, 65535]"
|
||||
store_url="List of stores to pack flists in which can be 'dir', 'zdb', 's3', required, example: ['dir:///tmp/store0']"
|
||||
flist_dir="A directory to save each user flists, required, example: 'flists'"
|
||||
|
||||
jwt_secret="secret for jwt, required, example: 'secret'"
|
||||
jwt_expire_hours="Life time for jwt token in hours, required, example: 5, validation: between [1, 24]"
|
||||
|
||||
[[users]] # list of authorized user in the server
|
||||
username = "user1"
|
||||
password = "password1"
|
||||
|
||||
[[users]]
|
||||
username = "user2"
|
||||
password = "password2"
|
||||
...
|
||||
```
|
||||
9
components/rfs/fl-server/build.rs
Normal file
9
components/rfs/fl-server/build.rs
Normal file
@@ -0,0 +1,9 @@
|
||||
fn main() {
|
||||
println!(
|
||||
"cargo:rustc-env=GIT_VERSION={}",
|
||||
git_version::git_version!(
|
||||
args = ["--tags", "--always", "--dirty=-modified"],
|
||||
fallback = "unknown"
|
||||
)
|
||||
);
|
||||
}
|
||||
154
components/rfs/fl-server/src/auth.rs
Normal file
154
components/rfs/fl-server/src/auth.rs
Normal file
@@ -0,0 +1,154 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
extract::{Json, Request, State},
|
||||
http::{self, StatusCode},
|
||||
middleware::Next,
|
||||
response::IntoResponse,
|
||||
};
|
||||
use axum_macros::debug_handler;
|
||||
use chrono::{Duration, Utc};
|
||||
use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::{
|
||||
config,
|
||||
response::{ResponseError, ResponseResult},
|
||||
};
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct Claims {
|
||||
pub exp: usize, // Expiry time of the token
|
||||
pub iat: usize, // Issued at time of the token
|
||||
pub username: String, // Username associated with the token
|
||||
}
|
||||
|
||||
#[derive(Deserialize, ToSchema)]
|
||||
pub struct SignInBody {
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub struct SignInResponse {
|
||||
pub access_token: String,
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/v1/api/signin",
|
||||
request_body = SignInBody,
|
||||
responses(
|
||||
(status = 200, description = "User signed in successfully", body = SignInResponse),
|
||||
(status = 500, description = "Internal server error"),
|
||||
(status = 401, description = "Unauthorized user"),
|
||||
)
|
||||
)]
|
||||
#[debug_handler]
|
||||
pub async fn sign_in_handler(
|
||||
State(state): State<Arc<config::AppState>>,
|
||||
Json(user_data): Json<SignInBody>,
|
||||
) -> impl IntoResponse {
|
||||
let user = match state.db.get_user_by_username(&user_data.username) {
|
||||
Some(user) => user,
|
||||
None => {
|
||||
return Err(ResponseError::Unauthorized(
|
||||
"User is not authorized".to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
if user_data.password != user.password {
|
||||
return Err(ResponseError::Unauthorized(
|
||||
"Wrong username or password".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let token = encode_jwt(
|
||||
user.username.clone(),
|
||||
state.config.jwt_secret.clone(),
|
||||
state.config.jwt_expire_hours,
|
||||
)
|
||||
.map_err(|_| ResponseError::InternalServerError)?;
|
||||
|
||||
Ok(ResponseResult::SignedIn(SignInResponse {
|
||||
access_token: token,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn encode_jwt(
|
||||
username: String,
|
||||
jwt_secret: String,
|
||||
jwt_expire: i64,
|
||||
) -> Result<String, StatusCode> {
|
||||
let now = Utc::now();
|
||||
let exp: usize = (now + Duration::hours(jwt_expire)).timestamp() as usize;
|
||||
let iat: usize = now.timestamp() as usize;
|
||||
let claim = Claims { iat, exp, username };
|
||||
|
||||
encode(
|
||||
&Header::default(),
|
||||
&claim,
|
||||
&EncodingKey::from_secret(jwt_secret.as_ref()),
|
||||
)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
|
||||
}
|
||||
|
||||
pub fn decode_jwt(jwt_token: String, jwt_secret: String) -> Result<TokenData<Claims>, StatusCode> {
|
||||
let result: Result<TokenData<Claims>, StatusCode> = decode(
|
||||
&jwt_token,
|
||||
&DecodingKey::from_secret(jwt_secret.as_ref()),
|
||||
&Validation::default(),
|
||||
)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR);
|
||||
result
|
||||
}
|
||||
|
||||
pub async fn authorize(
|
||||
State(state): State<Arc<config::AppState>>,
|
||||
mut req: Request,
|
||||
next: Next,
|
||||
) -> impl IntoResponse {
|
||||
let auth_header = match req.headers_mut().get(http::header::AUTHORIZATION) {
|
||||
Some(header) => header
|
||||
.to_str()
|
||||
.map_err(|_| ResponseError::Forbidden("Empty header is not allowed".to_string()))?,
|
||||
None => {
|
||||
return Err(ResponseError::Forbidden(
|
||||
"No JWT token is added to the header".to_string(),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let mut header = auth_header.split_whitespace();
|
||||
let (_, token) = (header.next(), header.next());
|
||||
let token_str = match token {
|
||||
Some(t) => t.to_string(),
|
||||
None => {
|
||||
log::error!("failed to get token string");
|
||||
return Err(ResponseError::InternalServerError);
|
||||
}
|
||||
};
|
||||
|
||||
let token_data = match decode_jwt(token_str, state.config.jwt_secret.clone()) {
|
||||
Ok(data) => data,
|
||||
Err(_) => {
|
||||
return Err(ResponseError::Forbidden(
|
||||
"Unable to decode JWT token".to_string(),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let current_user = match state.db.get_user_by_username(&token_data.claims.username) {
|
||||
Some(user) => user,
|
||||
None => {
|
||||
return Err(ResponseError::Unauthorized(
|
||||
"You are not an authorized user".to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
req.extensions_mut().insert(current_user.username.clone());
|
||||
Ok(next.run(req).await)
|
||||
}
|
||||
63
components/rfs/fl-server/src/config.rs
Normal file
63
components/rfs/fl-server/src/config.rs
Normal file
@@ -0,0 +1,63 @@
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::{
|
||||
db::{User, DB},
|
||||
handlers,
|
||||
};
|
||||
|
||||
#[derive(Debug, ToSchema, Serialize, Clone)]
|
||||
pub struct Job {
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
#[derive(ToSchema)]
|
||||
pub struct AppState {
|
||||
pub jobs_state: Mutex<HashMap<String, handlers::FlistState>>,
|
||||
pub flists_progress: Mutex<HashMap<PathBuf, f32>>,
|
||||
pub db: Arc<dyn DB>,
|
||||
pub config: Config,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct Config {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub store_url: Vec<String>,
|
||||
pub flist_dir: String,
|
||||
|
||||
pub jwt_secret: String,
|
||||
pub jwt_expire_hours: i64,
|
||||
pub users: Vec<User>,
|
||||
}
|
||||
|
||||
/// Parse the config file into Config struct.
|
||||
pub async fn parse_config(filepath: &str) -> Result<Config> {
|
||||
let content = fs::read_to_string(filepath).context("failed to read config file")?;
|
||||
let c: Config = toml::from_str(&content).context("failed to convert toml config data")?;
|
||||
|
||||
if !hostname_validator::is_valid(&c.host) {
|
||||
anyhow::bail!("host '{}' is invalid", c.host)
|
||||
}
|
||||
|
||||
rfs::store::parse_router(&c.store_url)
|
||||
.await
|
||||
.context("failed to parse store urls")?;
|
||||
fs::create_dir_all(&c.flist_dir).context("failed to create flists directory")?;
|
||||
|
||||
if c.jwt_expire_hours < 1 || c.jwt_expire_hours > 24 {
|
||||
anyhow::bail!(format!(
|
||||
"jwt expiry interval in hours '{}' is invalid, must be between [1, 24]",
|
||||
c.jwt_expire_hours
|
||||
))
|
||||
}
|
||||
|
||||
Ok(c)
|
||||
}
|
||||
36
components/rfs/fl-server/src/db.rs
Normal file
36
components/rfs/fl-server/src/db.rs
Normal file
@@ -0,0 +1,36 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct User {
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
}
|
||||
|
||||
pub trait DB: Send + Sync {
|
||||
fn get_user_by_username(&self, username: &str) -> Option<User>;
|
||||
}
|
||||
|
||||
#[derive(Debug, ToSchema)]
|
||||
pub struct MapDB {
|
||||
users: HashMap<String, User>,
|
||||
}
|
||||
|
||||
impl MapDB {
|
||||
pub fn new(users: &[User]) -> Self {
|
||||
Self {
|
||||
users: users
|
||||
.iter()
|
||||
.map(|u| (u.username.clone(), u.to_owned()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DB for MapDB {
|
||||
fn get_user_by_username(&self, username: &str) -> Option<User> {
|
||||
self.users.get(username).cloned()
|
||||
}
|
||||
}
|
||||
585
components/rfs/fl-server/src/handlers.rs
Normal file
585
components/rfs/fl-server/src/handlers.rs
Normal file
@@ -0,0 +1,585 @@
|
||||
use anyhow::Error;
|
||||
use axum::{
|
||||
extract::{Path, Query, State},
|
||||
response::IntoResponse,
|
||||
Extension, Json,
|
||||
};
|
||||
use axum_macros::debug_handler;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs,
|
||||
path::PathBuf,
|
||||
sync::{mpsc, Arc},
|
||||
};
|
||||
|
||||
use bollard::auth::DockerCredentials;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
auth::{SignInBody, SignInResponse, __path_sign_in_handler},
|
||||
response::{DirListTemplate, DirLister, ErrorTemplate, TemplateErr},
|
||||
};
|
||||
use crate::{
|
||||
config::{self, Job},
|
||||
response::{FileInfo, ResponseError, ResponseResult},
|
||||
serve_flists::visit_dir_one_level,
|
||||
};
|
||||
use rfs::fungi::{Reader, Writer};
|
||||
use utoipa::{OpenApi, ToSchema};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(OpenApi)]
|
||||
#[openapi(
|
||||
paths(health_check_handler, create_flist_handler, get_flist_state_handler, preview_flist_handler, list_flists_handler, sign_in_handler),
|
||||
components(schemas(DirListTemplate, DirLister, FlistBody, Job, ResponseError, ErrorTemplate, TemplateErr, ResponseResult, FileInfo, SignInBody, FlistState, SignInResponse, FlistStateInfo, PreviewResponse)),
|
||||
tags(
|
||||
(name = "fl-server", description = "Flist conversion API")
|
||||
)
|
||||
)]
|
||||
pub struct FlistApi;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, ToSchema)]
|
||||
pub struct FlistBody {
|
||||
#[schema(example = "redis")]
|
||||
pub image_name: String,
|
||||
|
||||
pub username: Option<String>,
|
||||
pub password: Option<String>,
|
||||
pub auth: Option<String>,
|
||||
pub email: Option<String>,
|
||||
pub server_address: Option<String>,
|
||||
pub identity_token: Option<String>,
|
||||
pub registry_token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, ToSchema)]
|
||||
pub struct PreviewResponse {
|
||||
pub content: Vec<PathBuf>,
|
||||
pub metadata: String,
|
||||
pub checksum: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, ToSchema)]
|
||||
pub enum FlistState {
|
||||
Accepted(String),
|
||||
Started(String),
|
||||
InProgress(FlistStateInfo),
|
||||
Created(String),
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, ToSchema)]
|
||||
pub struct FlistStateInfo {
|
||||
msg: String,
|
||||
progress: f32,
|
||||
}
|
||||
|
||||
const DEFAULT_LIMIT: usize = 10;
|
||||
const DEFAULT_PAGE: usize = 1;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Pagination {
|
||||
page: Option<usize>,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone)]
|
||||
pub struct Filter {
|
||||
pub max_size: Option<usize>,
|
||||
pub min_size: Option<usize>,
|
||||
username: Option<String>,
|
||||
pub name: Option<String>,
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/v1/api",
|
||||
responses(
|
||||
(status = 200, description = "flist server is working", body = String)
|
||||
)
|
||||
)]
|
||||
pub async fn health_check_handler() -> ResponseResult {
|
||||
ResponseResult::Health
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/v1/api/fl",
|
||||
request_body = FlistBody,
|
||||
responses(
|
||||
(status = 201, description = "Flist conversion started", body = Job),
|
||||
(status = 401, description = "Unauthorized user"),
|
||||
(status = 403, description = "Forbidden"),
|
||||
(status = 409, description = "Conflict"),
|
||||
(status = 500, description = "Internal server error"),
|
||||
)
|
||||
)]
|
||||
#[debug_handler]
|
||||
pub async fn create_flist_handler(
|
||||
State(state): State<Arc<config::AppState>>,
|
||||
Extension(username): Extension<String>,
|
||||
Json(body): Json<FlistBody>,
|
||||
) -> impl IntoResponse {
|
||||
let cfg = state.config.clone();
|
||||
let credentials = Some(DockerCredentials {
|
||||
username: body.username,
|
||||
password: body.password,
|
||||
auth: body.auth,
|
||||
email: body.email,
|
||||
serveraddress: body.server_address,
|
||||
identitytoken: body.identity_token,
|
||||
registrytoken: body.registry_token,
|
||||
});
|
||||
|
||||
let mut docker_image = body.image_name.to_string();
|
||||
if !docker_image.contains(':') {
|
||||
docker_image.push_str(":latest");
|
||||
}
|
||||
|
||||
let fl_name = docker_image.replace([':', '/'], "-") + ".fl";
|
||||
let username_dir = std::path::Path::new(&cfg.flist_dir).join(&username);
|
||||
let fl_path = username_dir.join(&fl_name);
|
||||
|
||||
if fl_path.exists() {
|
||||
return Err(ResponseError::Conflict("flist already exists".to_string()));
|
||||
}
|
||||
|
||||
if let Err(err) = fs::create_dir_all(&username_dir) {
|
||||
log::error!(
|
||||
"failed to create user flist directory `{:?}` with error {:?}",
|
||||
&username_dir,
|
||||
err
|
||||
);
|
||||
return Err(ResponseError::InternalServerError);
|
||||
}
|
||||
|
||||
let meta = match Writer::new(&fl_path, true).await {
|
||||
Ok(writer) => writer,
|
||||
Err(err) => {
|
||||
log::error!(
|
||||
"failed to create a new writer for flist `{:?}` with error {}",
|
||||
fl_path,
|
||||
err
|
||||
);
|
||||
return Err(ResponseError::InternalServerError);
|
||||
}
|
||||
};
|
||||
|
||||
let store = match rfs::store::parse_router(&cfg.store_url).await {
|
||||
Ok(s) => s,
|
||||
Err(err) => {
|
||||
log::error!("failed to parse router for store with error {}", err);
|
||||
return Err(ResponseError::InternalServerError);
|
||||
}
|
||||
};
|
||||
|
||||
// Create a new job id for the flist request
|
||||
let job: Job = Job {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
};
|
||||
let current_job = job.clone();
|
||||
|
||||
state
|
||||
.jobs_state
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.insert(
|
||||
job.id.clone(),
|
||||
FlistState::Accepted(format!("flist '{}' is accepted", &fl_name)),
|
||||
);
|
||||
|
||||
let flist_download_url = std::path::Path::new(&format!("{}:{}", cfg.host, cfg.port))
|
||||
.join(cfg.flist_dir)
|
||||
.join(username)
|
||||
.join(&fl_name);
|
||||
|
||||
tokio::spawn(async move {
|
||||
state
|
||||
.jobs_state
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.insert(
|
||||
job.id.clone(),
|
||||
FlistState::Started(format!("flist '{}' is started", fl_name)),
|
||||
);
|
||||
|
||||
let container_name = Uuid::new_v4().to_string();
|
||||
let docker_tmp_dir =
|
||||
tempdir::TempDir::new(&container_name).expect("failed to create tmp dir for docker");
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let mut docker_to_fl =
|
||||
docker2fl::DockerImageToFlist::new(meta, docker_image, credentials, docker_tmp_dir);
|
||||
|
||||
let res = docker_to_fl.prepare().await;
|
||||
if res.is_err() {
|
||||
let _ = tokio::fs::remove_file(&fl_path).await;
|
||||
state
|
||||
.jobs_state
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.insert(job.id.clone(), FlistState::Failed);
|
||||
return;
|
||||
}
|
||||
|
||||
let files_count = docker_to_fl.files_count();
|
||||
let st = state.clone();
|
||||
let job_id = job.id.clone();
|
||||
let cloned_fl_path = fl_path.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut progress: f32 = 0.0;
|
||||
|
||||
for _ in 0..files_count - 1 {
|
||||
let step = rx.recv().expect("failed to receive progress") as f32;
|
||||
progress += step;
|
||||
let progress_percentage = progress / files_count as f32 * 100.0;
|
||||
st.jobs_state.lock().expect("failed to lock state").insert(
|
||||
job_id.clone(),
|
||||
FlistState::InProgress(FlistStateInfo {
|
||||
msg: "flist is in progress".to_string(),
|
||||
progress: progress_percentage,
|
||||
}),
|
||||
);
|
||||
st.flists_progress
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.insert(cloned_fl_path.clone(), progress_percentage);
|
||||
}
|
||||
});
|
||||
|
||||
let res = docker_to_fl.pack(store, Some(tx)).await;
|
||||
|
||||
// remove the file created with the writer if fl creation failed
|
||||
if res.is_err() {
|
||||
log::error!("failed creation failed with error {:?}", res.err());
|
||||
let _ = tokio::fs::remove_file(&fl_path).await;
|
||||
state
|
||||
.jobs_state
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.insert(job.id.clone(), FlistState::Failed);
|
||||
return;
|
||||
}
|
||||
|
||||
state
|
||||
.jobs_state
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.insert(
|
||||
job.id.clone(),
|
||||
FlistState::Created(format!(
|
||||
"flist {:?} is created successfully",
|
||||
flist_download_url
|
||||
)),
|
||||
);
|
||||
state
|
||||
.flists_progress
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.insert(fl_path, 100.0);
|
||||
});
|
||||
|
||||
Ok(ResponseResult::FlistCreated(current_job))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/v1/api/fl/{job_id}",
|
||||
responses(
|
||||
(status = 200, description = "Flist state", body = FlistState),
|
||||
(status = 404, description = "Flist not found"),
|
||||
(status = 500, description = "Internal server error"),
|
||||
(status = 401, description = "Unauthorized user"),
|
||||
(status = 403, description = "Forbidden"),
|
||||
),
|
||||
params(
|
||||
("job_id" = String, Path, description = "flist job id")
|
||||
)
|
||||
)]
|
||||
#[debug_handler]
|
||||
pub async fn get_flist_state_handler(
|
||||
Path(flist_job_id): Path<String>,
|
||||
State(state): State<Arc<config::AppState>>,
|
||||
) -> impl IntoResponse {
|
||||
if !&state
|
||||
.jobs_state
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.contains_key(&flist_job_id.clone())
|
||||
{
|
||||
return Err(ResponseError::NotFound("flist doesn't exist".to_string()));
|
||||
}
|
||||
|
||||
let res_state = state
|
||||
.jobs_state
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.get(&flist_job_id.clone())
|
||||
.expect("failed to get from state")
|
||||
.to_owned();
|
||||
|
||||
match res_state {
|
||||
FlistState::Accepted(_) => Ok(ResponseResult::FlistState(res_state)),
|
||||
FlistState::Started(_) => Ok(ResponseResult::FlistState(res_state)),
|
||||
FlistState::InProgress(_) => Ok(ResponseResult::FlistState(res_state)),
|
||||
FlistState::Created(_) => {
|
||||
state
|
||||
.jobs_state
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.remove(&flist_job_id.clone());
|
||||
|
||||
Ok(ResponseResult::FlistState(res_state))
|
||||
}
|
||||
FlistState::Failed => {
|
||||
state
|
||||
.jobs_state
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.remove(&flist_job_id.clone());
|
||||
|
||||
Err(ResponseError::InternalServerError)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/v1/api/fl",
|
||||
responses(
|
||||
(status = 200, description = "Listing flists", body = HashMap<String, Vec<FileInfo>>),
|
||||
(status = 401, description = "Unauthorized user"),
|
||||
(status = 403, description = "Forbidden"),
|
||||
(status = 500, description = "Internal server error"),
|
||||
)
|
||||
)]
|
||||
#[debug_handler]
|
||||
pub async fn list_flists_handler(
|
||||
State(state): State<Arc<config::AppState>>,
|
||||
pagination: Query<Pagination>,
|
||||
filter: Query<Filter>,
|
||||
) -> impl IntoResponse {
|
||||
let mut flists: HashMap<String, Vec<FileInfo>> = HashMap::new();
|
||||
|
||||
let pagination: Pagination = pagination.0;
|
||||
let page = pagination.page.unwrap_or(DEFAULT_PAGE);
|
||||
let limit = pagination.limit.unwrap_or(DEFAULT_LIMIT);
|
||||
|
||||
if page == 0 {
|
||||
return Err(ResponseError::BadRequest(
|
||||
"requested page should be nonzero positive number".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let filter: Filter = filter.0;
|
||||
|
||||
let rs: Result<Vec<FileInfo>, std::io::Error> =
|
||||
visit_dir_one_level(&state.config.flist_dir, &state, None).await;
|
||||
|
||||
let files = match rs {
|
||||
Ok(files) => files,
|
||||
Err(e) => {
|
||||
log::error!("failed to list flists directory with error: {}", e);
|
||||
return Err(ResponseError::InternalServerError);
|
||||
}
|
||||
};
|
||||
|
||||
for file in files {
|
||||
if !file.is_file {
|
||||
let flists_per_username =
|
||||
visit_dir_one_level(&file.path_uri, &state, Some(filter.clone())).await;
|
||||
|
||||
if let Some(ref filter_username) = filter.username {
|
||||
if filter_username.clone() != file.name {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
match flists_per_username {
|
||||
Ok(files) => {
|
||||
let username = file.name;
|
||||
flists.insert(username.clone(), Vec::new());
|
||||
|
||||
let start = limit * (page - 1);
|
||||
let end = limit * page;
|
||||
if files.len() > start {
|
||||
if files.len() >= end {
|
||||
flists.insert(username, files[start..end].to_vec());
|
||||
} else {
|
||||
flists.insert(username, files[start..].to_vec());
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("failed to list flists per username with error: {}", e);
|
||||
return Err(ResponseError::InternalServerError);
|
||||
}
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
Ok(ResponseResult::Flists(flists))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/v1/api/fl/preview/{flist_path}",
|
||||
responses(
|
||||
(status = 200, description = "Flist preview result", body = PreviewResponse),
|
||||
(status = 400, description = "Bad request"),
|
||||
(status = 401, description = "Unauthorized user"),
|
||||
(status = 403, description = "Forbidden"),
|
||||
(status = 500, description = "Internal server error"),
|
||||
),
|
||||
params(
|
||||
("flist_path" = String, Path, description = "flist file path")
|
||||
)
|
||||
)]
|
||||
#[debug_handler]
|
||||
pub async fn preview_flist_handler(
|
||||
State(state): State<Arc<config::AppState>>,
|
||||
Path(flist_path): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
let fl_path = flist_path;
|
||||
|
||||
match validate_flist_path(&state, &fl_path).await {
|
||||
Ok(_) => (),
|
||||
Err(err) => return Err(ResponseError::BadRequest(err.to_string())),
|
||||
};
|
||||
|
||||
let content = match get_flist_content(&fl_path).await {
|
||||
Ok(paths) => paths,
|
||||
Err(_) => return Err(ResponseError::InternalServerError),
|
||||
};
|
||||
|
||||
let bytes = match std::fs::read(&fl_path) {
|
||||
Ok(b) => b,
|
||||
Err(err) => {
|
||||
log::error!(
|
||||
"failed to read flist '{}' into bytes with error {}",
|
||||
fl_path,
|
||||
err
|
||||
);
|
||||
return Err(ResponseError::InternalServerError);
|
||||
}
|
||||
};
|
||||
|
||||
Ok(ResponseResult::PreviewFlist(PreviewResponse {
|
||||
content,
|
||||
metadata: state.config.store_url.join("-"),
|
||||
checksum: sha256::digest(&bytes),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn validate_flist_path(state: &Arc<config::AppState>, fl_path: &String) -> Result<(), Error> {
|
||||
// validate path starting with `/`
|
||||
if fl_path.starts_with("/") {
|
||||
anyhow::bail!("invalid flist path '{}', shouldn't start with '/'", fl_path);
|
||||
}
|
||||
|
||||
// path should include 3 parts [parent dir, username, flist file]
|
||||
let parts: Vec<_> = fl_path.split("/").collect();
|
||||
if parts.len() != 3 {
|
||||
anyhow::bail!(
|
||||
format!("invalid flist path '{}', should consist of 3 parts [parent directory, username and flist name", fl_path
|
||||
));
|
||||
}
|
||||
|
||||
// validate parent dir
|
||||
if parts[0] != state.config.flist_dir {
|
||||
anyhow::bail!(
|
||||
"invalid flist path '{}', parent directory should be '{}'",
|
||||
fl_path,
|
||||
state.config.flist_dir
|
||||
);
|
||||
}
|
||||
|
||||
// validate username
|
||||
match state.db.get_user_by_username(&parts[1]) {
|
||||
Some(_) => (),
|
||||
None => {
|
||||
anyhow::bail!(
|
||||
"invalid flist path '{}', username '{}' doesn't exist",
|
||||
fl_path,
|
||||
parts[1]
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// validate flist extension
|
||||
let fl_name = parts[2].to_string();
|
||||
let ext = match std::path::Path::new(&fl_name).extension() {
|
||||
Some(ex) => ex.to_string_lossy().to_string(),
|
||||
None => "".to_string(),
|
||||
};
|
||||
|
||||
if ext != "fl" {
|
||||
anyhow::bail!(
|
||||
"invalid flist path '{}', invalid flist extension '{}' should be 'fl'",
|
||||
fl_path,
|
||||
ext
|
||||
);
|
||||
}
|
||||
|
||||
// validate flist existence
|
||||
if !std::path::Path::new(parts[0])
|
||||
.join(parts[1])
|
||||
.join(&fl_name)
|
||||
.exists()
|
||||
{
|
||||
anyhow::bail!("flist '{}' doesn't exist", fl_path);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_flist_content(fl_path: &String) -> Result<Vec<PathBuf>, Error> {
|
||||
let mut visitor = ReadVisitor::default();
|
||||
|
||||
let meta = match Reader::new(&fl_path).await {
|
||||
Ok(reader) => reader,
|
||||
Err(err) => {
|
||||
log::error!(
|
||||
"failed to initialize metadata database for flist `{}` with error {}",
|
||||
fl_path,
|
||||
err
|
||||
);
|
||||
anyhow::bail!("Internal server error");
|
||||
}
|
||||
};
|
||||
|
||||
match meta.walk(&mut visitor).await {
|
||||
Ok(()) => return Ok(visitor.into_inner()),
|
||||
Err(err) => {
|
||||
log::error!(
|
||||
"failed to walk through metadata for flist `{}` with error {}",
|
||||
fl_path,
|
||||
err
|
||||
);
|
||||
anyhow::bail!("Internal server error");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReadVisitor {
|
||||
inner: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
impl ReadVisitor {
|
||||
pub fn into_inner(self) -> Vec<PathBuf> {
|
||||
self.inner
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl rfs::fungi::meta::WalkVisitor for ReadVisitor {
|
||||
async fn visit(
|
||||
&mut self,
|
||||
path: &std::path::Path,
|
||||
_node: &rfs::fungi::meta::Inode,
|
||||
) -> rfs::fungi::meta::Result<rfs::fungi::meta::Walk> {
|
||||
self.inner.push(path.to_path_buf());
|
||||
Ok(rfs::fungi::meta::Walk::Continue)
|
||||
}
|
||||
}
|
||||
186
components/rfs/fl-server/src/main.rs
Normal file
186
components/rfs/fl-server/src/main.rs
Normal file
@@ -0,0 +1,186 @@
|
||||
mod auth;
|
||||
mod config;
|
||||
mod db;
|
||||
mod handlers;
|
||||
mod response;
|
||||
mod serve_flists;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use axum::{
|
||||
error_handling::HandleErrorLayer,
|
||||
http::StatusCode,
|
||||
middleware,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
BoxError, Router,
|
||||
};
|
||||
use clap::{ArgAction, Parser};
|
||||
use hyper::{
|
||||
header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE},
|
||||
Method,
|
||||
};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{runtime::Builder, signal};
|
||||
use tower::ServiceBuilder;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tower_http::{cors::Any, trace::TraceLayer};
|
||||
|
||||
use utoipa::OpenApi;
|
||||
use utoipa_swagger_ui::SwaggerUi;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(name ="fl-server", author, version = env!("GIT_VERSION"), about, long_about = None)]
|
||||
struct Options {
|
||||
/// enable debugging logs
|
||||
#[clap(short, long, action=ArgAction::Count)]
|
||||
debug: u8,
|
||||
|
||||
/// config file path
|
||||
#[clap(short, long)]
|
||||
config_path: String,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let rt = Builder::new_multi_thread()
|
||||
.thread_stack_size(8 * 1024 * 1024)
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap();
|
||||
rt.block_on(app())
|
||||
}
|
||||
|
||||
async fn app() -> Result<()> {
|
||||
let opts = Options::parse();
|
||||
simple_logger::SimpleLogger::new()
|
||||
.with_utc_timestamps()
|
||||
.with_level({
|
||||
match opts.debug {
|
||||
0 => log::LevelFilter::Info,
|
||||
1 => log::LevelFilter::Debug,
|
||||
_ => log::LevelFilter::Trace,
|
||||
}
|
||||
})
|
||||
.with_module_level("sqlx", log::Level::Error.to_level_filter())
|
||||
.init()?;
|
||||
|
||||
let config = config::parse_config(&opts.config_path)
|
||||
.await
|
||||
.context("failed to parse config file")?;
|
||||
|
||||
let db = Arc::new(db::MapDB::new(&config.users.clone()));
|
||||
|
||||
let app_state = Arc::new(config::AppState {
|
||||
jobs_state: Mutex::new(HashMap::new()),
|
||||
flists_progress: Mutex::new(HashMap::new()),
|
||||
db,
|
||||
config,
|
||||
});
|
||||
|
||||
let cors = CorsLayer::new()
|
||||
.allow_origin(Any)
|
||||
.allow_methods([Method::GET, Method::POST])
|
||||
.allow_headers([AUTHORIZATION, ACCEPT, CONTENT_TYPE]);
|
||||
|
||||
let v1_routes = Router::new()
|
||||
.route("/v1/api", get(handlers::health_check_handler))
|
||||
.route("/v1/api/signin", post(auth::sign_in_handler))
|
||||
.route(
|
||||
"/v1/api/fl",
|
||||
post(handlers::create_flist_handler).layer(middleware::from_fn_with_state(
|
||||
app_state.clone(),
|
||||
auth::authorize,
|
||||
)),
|
||||
)
|
||||
.route(
|
||||
"/v1/api/fl/:job_id",
|
||||
get(handlers::get_flist_state_handler).layer(middleware::from_fn_with_state(
|
||||
app_state.clone(),
|
||||
auth::authorize,
|
||||
)),
|
||||
)
|
||||
.route(
|
||||
"/v1/api/fl/preview/:flist_path",
|
||||
get(handlers::preview_flist_handler),
|
||||
)
|
||||
.route("/v1/api/fl", get(handlers::list_flists_handler))
|
||||
.route("/*path", get(serve_flists::serve_flists));
|
||||
|
||||
let app = Router::new()
|
||||
.merge(
|
||||
SwaggerUi::new("/swagger-ui")
|
||||
.url("/api-docs/openapi.json", handlers::FlistApi::openapi()),
|
||||
)
|
||||
.merge(v1_routes)
|
||||
.layer(
|
||||
ServiceBuilder::new()
|
||||
.layer(HandleErrorLayer::new(handle_error))
|
||||
.load_shed()
|
||||
.concurrency_limit(1024)
|
||||
.timeout(Duration::from_secs(10))
|
||||
.layer(TraceLayer::new_for_http()),
|
||||
)
|
||||
.with_state(Arc::clone(&app_state))
|
||||
.layer(cors);
|
||||
|
||||
let address = format!("{}:{}", app_state.config.host, app_state.config.port);
|
||||
let listener = tokio::net::TcpListener::bind(address)
|
||||
.await
|
||||
.context("failed to bind address")?;
|
||||
|
||||
log::info!(
|
||||
"🚀 Server started successfully at {}:{}",
|
||||
app_state.config.host,
|
||||
app_state.config.port
|
||||
);
|
||||
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.await
|
||||
.context("failed to serve listener")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
signal::ctrl_c()
|
||||
.await
|
||||
.expect("failed to install Ctrl+C handler");
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
signal::unix::signal(signal::unix::SignalKind::terminate())
|
||||
.expect("failed to install signal handler")
|
||||
.recv()
|
||||
.await;
|
||||
};
|
||||
|
||||
tokio::select! {
|
||||
_ = ctrl_c => {},
|
||||
_ = terminate => {},
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_error(error: BoxError) -> impl IntoResponse {
|
||||
if error.is::<tower::timeout::error::Elapsed>() {
|
||||
return (StatusCode::REQUEST_TIMEOUT, Cow::from("request timed out"));
|
||||
}
|
||||
|
||||
if error.is::<tower::load_shed::error::Overloaded>() {
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Cow::from("service is overloaded, try again later"),
|
||||
);
|
||||
}
|
||||
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Cow::from(format!("Unhandled internal error: {}", error)),
|
||||
)
|
||||
}
|
||||
178
components/rfs/fl-server/src/response.rs
Normal file
178
components/rfs/fl-server/src/response.rs
Normal file
@@ -0,0 +1,178 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use askama::Template;
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::StatusCode,
|
||||
response::{Html, IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde::Serialize;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::{
|
||||
auth::SignInResponse,
|
||||
config::Job,
|
||||
handlers::{FlistState, PreviewResponse},
|
||||
};
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub enum ResponseError {
|
||||
InternalServerError,
|
||||
Conflict(String),
|
||||
NotFound(String),
|
||||
Unauthorized(String),
|
||||
BadRequest(String),
|
||||
Forbidden(String),
|
||||
TemplateError(ErrorTemplate),
|
||||
}
|
||||
|
||||
impl IntoResponse for ResponseError {
|
||||
fn into_response(self) -> Response<Body> {
|
||||
match self {
|
||||
ResponseError::InternalServerError => {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, "Internal server error").into_response()
|
||||
}
|
||||
ResponseError::Conflict(msg) => (StatusCode::CONFLICT, msg).into_response(),
|
||||
ResponseError::NotFound(msg) => (StatusCode::NOT_FOUND, msg).into_response(),
|
||||
ResponseError::Unauthorized(msg) => (StatusCode::UNAUTHORIZED, msg).into_response(),
|
||||
ResponseError::BadRequest(msg) => (StatusCode::BAD_REQUEST, msg).into_response(),
|
||||
ResponseError::Forbidden(msg) => (StatusCode::FORBIDDEN, msg).into_response(),
|
||||
ResponseError::TemplateError(t) => match t.render() {
|
||||
Ok(html) => {
|
||||
let mut resp = Html(html).into_response();
|
||||
match t.err {
|
||||
TemplateErr::NotFound(reason) => {
|
||||
*resp.status_mut() = StatusCode::NOT_FOUND;
|
||||
resp.headers_mut()
|
||||
.insert(FAIL_REASON_HEADER_NAME, reason.parse().unwrap());
|
||||
}
|
||||
TemplateErr::BadRequest(reason) => {
|
||||
*resp.status_mut() = StatusCode::BAD_REQUEST;
|
||||
resp.headers_mut()
|
||||
.insert(FAIL_REASON_HEADER_NAME, reason.parse().unwrap());
|
||||
}
|
||||
TemplateErr::InternalServerError(reason) => {
|
||||
*resp.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
|
||||
resp.headers_mut()
|
||||
.insert(FAIL_REASON_HEADER_NAME, reason.parse().unwrap());
|
||||
}
|
||||
}
|
||||
resp
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::error!("template render failed, err={}", err);
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("Failed to render template. Error: {}", err),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(ToSchema)]
|
||||
pub enum ResponseResult {
|
||||
Health,
|
||||
FlistCreated(Job),
|
||||
FlistState(FlistState),
|
||||
Flists(HashMap<String, Vec<FileInfo>>),
|
||||
PreviewFlist(PreviewResponse),
|
||||
SignedIn(SignInResponse),
|
||||
DirTemplate(DirListTemplate),
|
||||
Res(hyper::Response<tower_http::services::fs::ServeFileSystemResponseBody>),
|
||||
}
|
||||
|
||||
impl IntoResponse for ResponseResult {
|
||||
fn into_response(self) -> Response<Body> {
|
||||
match self {
|
||||
ResponseResult::Health => (
|
||||
StatusCode::OK,
|
||||
Json(serde_json::json!({"msg": "flist server is working"})),
|
||||
)
|
||||
.into_response(),
|
||||
ResponseResult::SignedIn(token) => (StatusCode::CREATED, Json(token)).into_response(),
|
||||
ResponseResult::FlistCreated(job) => (StatusCode::CREATED, Json(job)).into_response(),
|
||||
ResponseResult::FlistState(flist_state) => (
|
||||
StatusCode::OK,
|
||||
Json(serde_json::json!({
|
||||
"flist_state": flist_state
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
ResponseResult::Flists(flists) => (StatusCode::OK, Json(flists)).into_response(),
|
||||
ResponseResult::PreviewFlist(content) => {
|
||||
(StatusCode::OK, Json(content)).into_response()
|
||||
}
|
||||
ResponseResult::DirTemplate(t) => match t.render() {
|
||||
Ok(html) => Html(html).into_response(),
|
||||
Err(err) => {
|
||||
tracing::error!("template render failed, err={}", err);
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("Failed to render template. Error: {}", err),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
},
|
||||
ResponseResult::Res(res) => res.map(axum::body::Body::new),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//////// TEMPLATES ////////
|
||||
|
||||
#[derive(Serialize, Clone, Debug, ToSchema)]
|
||||
pub struct FileInfo {
|
||||
pub name: String,
|
||||
pub path_uri: String,
|
||||
pub is_file: bool,
|
||||
pub size: u64,
|
||||
pub last_modified: i64,
|
||||
pub progress: f32,
|
||||
}
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub struct DirLister {
|
||||
pub files: Vec<FileInfo>,
|
||||
}
|
||||
|
||||
#[derive(Template, Serialize, ToSchema)]
|
||||
#[template(path = "index.html")]
|
||||
pub struct DirListTemplate {
|
||||
pub lister: DirLister,
|
||||
pub cur_path: String,
|
||||
}
|
||||
|
||||
mod filters {
|
||||
pub(crate) fn datetime(ts: &i64) -> ::askama::Result<String> {
|
||||
if let Ok(format) =
|
||||
time::format_description::parse("[year]-[month]-[day] [hour]:[minute]:[second] UTC")
|
||||
{
|
||||
return Ok(time::OffsetDateTime::from_unix_timestamp(*ts)
|
||||
.unwrap()
|
||||
.format(&format)
|
||||
.unwrap());
|
||||
}
|
||||
Err(askama::Error::Fmt(std::fmt::Error))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Template, Serialize, ToSchema)]
|
||||
#[template(path = "error.html")]
|
||||
pub struct ErrorTemplate {
|
||||
pub err: TemplateErr,
|
||||
pub cur_path: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
const FAIL_REASON_HEADER_NAME: &str = "fl-server-fail-reason";
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub enum TemplateErr {
|
||||
BadRequest(String),
|
||||
NotFound(String),
|
||||
InternalServerError(String),
|
||||
}
|
||||
174
components/rfs/fl-server/src/serve_flists.rs
Normal file
174
components/rfs/fl-server/src/serve_flists.rs
Normal file
@@ -0,0 +1,174 @@
|
||||
use axum::extract::State;
|
||||
use std::{io::Error, path::PathBuf, sync::Arc};
|
||||
use tokio::io;
|
||||
use tower::util::ServiceExt;
|
||||
use tower_http::services::ServeDir;
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode},
|
||||
response::IntoResponse,
|
||||
};
|
||||
use axum_macros::debug_handler;
|
||||
use percent_encoding::percent_decode;
|
||||
|
||||
use crate::{
|
||||
config,
|
||||
handlers::Filter,
|
||||
response::{
|
||||
DirListTemplate, DirLister, ErrorTemplate, FileInfo, ResponseError, ResponseResult,
|
||||
TemplateErr,
|
||||
},
|
||||
};
|
||||
|
||||
#[debug_handler]
|
||||
pub async fn serve_flists(
|
||||
State(state): State<Arc<config::AppState>>,
|
||||
req: Request<Body>,
|
||||
) -> impl IntoResponse {
|
||||
let path = req.uri().path().to_string();
|
||||
|
||||
return match ServeDir::new("").oneshot(req).await {
|
||||
Ok(res) => {
|
||||
let status = res.status();
|
||||
match status {
|
||||
StatusCode::NOT_FOUND => {
|
||||
let full_path = match validate_path(&path) {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
return Err(ResponseError::TemplateError(ErrorTemplate {
|
||||
err: TemplateErr::BadRequest("invalid path".to_string()),
|
||||
cur_path: path.to_string(),
|
||||
message: "invalid path".to_owned(),
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
let cur_path = std::path::Path::new(&full_path);
|
||||
|
||||
match cur_path.is_dir() {
|
||||
true => {
|
||||
let rs = visit_dir_one_level(&full_path, &state, None).await;
|
||||
match rs {
|
||||
Ok(files) => Ok(ResponseResult::DirTemplate(DirListTemplate {
|
||||
lister: DirLister { files },
|
||||
cur_path: path.to_string(),
|
||||
})),
|
||||
Err(e) => Err(ResponseError::TemplateError(ErrorTemplate {
|
||||
err: TemplateErr::InternalServerError(e.to_string()),
|
||||
cur_path: path.to_string(),
|
||||
message: e.to_string(),
|
||||
})),
|
||||
}
|
||||
}
|
||||
false => Err(ResponseError::TemplateError(ErrorTemplate {
|
||||
err: TemplateErr::NotFound("file not found".to_string()),
|
||||
cur_path: path.to_string(),
|
||||
message: "file not found".to_owned(),
|
||||
})),
|
||||
}
|
||||
}
|
||||
_ => Ok(ResponseResult::Res(res)),
|
||||
}
|
||||
}
|
||||
Err(err) => Err(ResponseError::TemplateError(ErrorTemplate {
|
||||
err: TemplateErr::InternalServerError(format!("Unhandled error: {}", err)),
|
||||
cur_path: path.to_string(),
|
||||
message: format!("Unhandled error: {}", err),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
fn validate_path(path: &str) -> io::Result<PathBuf> {
|
||||
let path = path.trim_start_matches('/');
|
||||
let path = percent_decode(path.as_ref()).decode_utf8_lossy();
|
||||
|
||||
let mut full_path = PathBuf::new();
|
||||
|
||||
// validate
|
||||
for seg in path.split('/') {
|
||||
if seg.starts_with("..") || seg.contains('\\') {
|
||||
return Err(Error::other("invalid path"));
|
||||
}
|
||||
full_path.push(seg);
|
||||
}
|
||||
|
||||
Ok(full_path)
|
||||
}
|
||||
|
||||
pub async fn visit_dir_one_level<P: AsRef<std::path::Path>>(
|
||||
path: P,
|
||||
state: &Arc<config::AppState>,
|
||||
filter: Option<Filter>,
|
||||
) -> io::Result<Vec<FileInfo>> {
|
||||
let path = path.as_ref();
|
||||
let mut dir = tokio::fs::read_dir(path).await?;
|
||||
let mut files: Vec<FileInfo> = Vec::new();
|
||||
|
||||
while let Some(child) = dir.next_entry().await? {
|
||||
let path_uri = child.path().to_string_lossy().to_string();
|
||||
let is_file = child.file_type().await?.is_file();
|
||||
let name = child.file_name().to_string_lossy().to_string();
|
||||
let size = child.metadata().await?.len();
|
||||
|
||||
let mut progress = 0.0;
|
||||
if is_file {
|
||||
match state
|
||||
.flists_progress
|
||||
.lock()
|
||||
.expect("failed to lock state")
|
||||
.get(&path.join(&name).to_path_buf())
|
||||
{
|
||||
Some(p) => progress = *p,
|
||||
None => progress = 100.0,
|
||||
}
|
||||
|
||||
let ext = child
|
||||
.path()
|
||||
.extension()
|
||||
.expect("failed to get path extension")
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
if ext != "fl" {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref filter_files) = filter {
|
||||
if let Some(ref filter_name) = filter_files.name {
|
||||
if filter_name.clone() != name {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref filter_max_size) = filter_files.max_size {
|
||||
if filter_max_size.clone() < size as usize {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref filter_min_size) = filter_files.min_size {
|
||||
if filter_min_size.clone() > size as usize {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
files.push(FileInfo {
|
||||
name,
|
||||
path_uri,
|
||||
is_file,
|
||||
size: size,
|
||||
last_modified: child
|
||||
.metadata()
|
||||
.await?
|
||||
.modified()?
|
||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)
|
||||
.expect("failed to get duration")
|
||||
.as_secs() as i64,
|
||||
progress,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
57
components/rfs/fl-server/templates/base.html
Normal file
57
components/rfs/fl-server/templates/base.html
Normal file
@@ -0,0 +1,57 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>{% block title %}{{ title }}{% endblock %}</title>
|
||||
|
||||
<style>
|
||||
a {
|
||||
text-decoration: none;
|
||||
color: #4c7883;
|
||||
cursor: pointer;
|
||||
}
|
||||
a:hover, a:hover .directory, a:hover .file {
|
||||
color: #6ea90c;
|
||||
}
|
||||
|
||||
.directory {
|
||||
color:#6ea90c;
|
||||
}
|
||||
|
||||
.file {
|
||||
color: #585a56;
|
||||
}
|
||||
|
||||
#main {
|
||||
width: 60%;
|
||||
max-width: 1600px;
|
||||
margin: 1em auto;
|
||||
}
|
||||
|
||||
footer {
|
||||
display: block;
|
||||
padding: 40px;
|
||||
font-size: 12px;
|
||||
text-align: center;
|
||||
}
|
||||
</style>
|
||||
{% block head %}{% endblock %}
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div id="main">
|
||||
{% block content %}{% endblock %}
|
||||
</div>
|
||||
|
||||
<footer>
|
||||
Served with <a href="https://github.com/threefoldtech/rfs">flist-server</a>
|
||||
</footer>
|
||||
|
||||
{% block foot %}{% endblock %}
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
|
||||
|
||||
16
components/rfs/fl-server/templates/error.html
Normal file
16
components/rfs/fl-server/templates/error.html
Normal file
@@ -0,0 +1,16 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Error Occured when listing directory for {{ cur_path }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Error</h1>
|
||||
|
||||
<hr>
|
||||
|
||||
<dl>
|
||||
<dt> Request Path: </dt> <dd> <span class="directory">{{ cur_path }}</span> </dd>
|
||||
<dt> Error Message: </dt> <dd> <span class="error">{{ message }} </span> </dd>
|
||||
</dl>
|
||||
|
||||
<hr />
|
||||
{% endblock %}
|
||||
41
components/rfs/fl-server/templates/index.html
Normal file
41
components/rfs/fl-server/templates/index.html
Normal file
@@ -0,0 +1,41 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Directory listing for /{{ cur_path }}{% endblock %}
|
||||
|
||||
{% block head %}
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
|
||||
{% endblock %}
|
||||
|
||||
|
||||
{% block content %}
|
||||
<h1>Directory listing for <span class="directory">/{{ cur_path }}</span></h1>
|
||||
<hr>
|
||||
|
||||
<ol>
|
||||
{% for file in lister.files %}
|
||||
|
||||
{% if file.is_file %}
|
||||
<li class="item">
|
||||
<a class="file"
|
||||
href="/{{ file.path_uri|urlencode }}"
|
||||
data-src="{{ file.path_uri|urlencode }}"
|
||||
title="{{file.name}} {{ file.last_modified|datetime }}">
|
||||
|
||||
<span class="fa fa-file"></span> {{file.name}}
|
||||
</a>
|
||||
</li>
|
||||
|
||||
{% else %}
|
||||
<li>
|
||||
<a href="/{{ file.path_uri|urlencode }}/" title="{{ file.last_modified|datetime }}">
|
||||
<span class="fa fa-folder"></span> {{file.name}}/
|
||||
</a>
|
||||
</li>
|
||||
{% endif %}
|
||||
|
||||
{% endfor %}
|
||||
</ol>
|
||||
|
||||
<hr>
|
||||
|
||||
{% endblock %}
|
||||
Reference in New Issue
Block a user