Squashed 'components/rfs/' content from commit 9808a5e
git-subtree-dir: components/rfs git-subtree-split: 9808a5e9fc768edc7d8b1dfa5b91b3f018dff0cb
This commit is contained in:
335
docker2fl/src/docker2fl.rs
Normal file
335
docker2fl/src/docker2fl.rs
Normal file
@@ -0,0 +1,335 @@
|
||||
use bollard::auth::DockerCredentials;
|
||||
use bollard::container::{
|
||||
Config, CreateContainerOptions, InspectContainerOptions, RemoveContainerOptions,
|
||||
};
|
||||
use bollard::image::{CreateImageOptions, RemoveImageOptions};
|
||||
use bollard::Docker;
|
||||
use std::sync::mpsc::Sender;
|
||||
use tempdir::TempDir;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use futures_util::stream::StreamExt;
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::default::Default;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use tokio_async_drop::tokio_async_drop;
|
||||
|
||||
use rfs::fungi::Writer;
|
||||
use rfs::store::Store;
|
||||
|
||||
struct DockerInfo {
|
||||
image_name: String,
|
||||
container_name: String,
|
||||
docker: Docker,
|
||||
}
|
||||
|
||||
impl Drop for DockerInfo {
|
||||
fn drop(&mut self) {
|
||||
tokio_async_drop!({
|
||||
let res = clean(&self.docker, &self.image_name, &self.container_name)
|
||||
.await
|
||||
.context("failed to clean docker image and container");
|
||||
|
||||
if res.is_err() {
|
||||
log::error!(
|
||||
"cleaning docker image and container failed with error: {:?}",
|
||||
res.err()
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DockerImageToFlist {
|
||||
meta: Writer,
|
||||
image_name: String,
|
||||
credentials: Option<DockerCredentials>,
|
||||
docker_tmp_dir: TempDir,
|
||||
}
|
||||
|
||||
impl DockerImageToFlist {
|
||||
pub fn new(
|
||||
meta: Writer,
|
||||
image_name: String,
|
||||
credentials: Option<DockerCredentials>,
|
||||
docker_tmp_dir: TempDir,
|
||||
) -> Self {
|
||||
DockerImageToFlist {
|
||||
meta,
|
||||
image_name,
|
||||
credentials,
|
||||
docker_tmp_dir,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn files_count(&self) -> usize {
|
||||
WalkDir::new(self.docker_tmp_dir.path()).into_iter().count()
|
||||
}
|
||||
|
||||
pub async fn prepare(&mut self) -> Result<()> {
|
||||
#[cfg(unix)]
|
||||
let docker = Docker::connect_with_socket_defaults().context("failed to create docker")?;
|
||||
|
||||
let container_file =
|
||||
Path::file_stem(self.docker_tmp_dir.path()).expect("failed to get directory name");
|
||||
let container_name = container_file
|
||||
.to_str()
|
||||
.expect("failed to get container name")
|
||||
.to_owned();
|
||||
|
||||
let docker_info = DockerInfo {
|
||||
image_name: self.image_name.to_owned(),
|
||||
container_name,
|
||||
docker,
|
||||
};
|
||||
|
||||
extract_image(
|
||||
&docker_info.docker,
|
||||
&docker_info.image_name,
|
||||
&docker_info.container_name,
|
||||
self.docker_tmp_dir.path(),
|
||||
self.credentials.clone(),
|
||||
)
|
||||
.await
|
||||
.context("failed to extract docker image to a directory")?;
|
||||
log::info!(
|
||||
"docker image '{}' is extracted successfully",
|
||||
docker_info.image_name
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn pack<S: Store>(&mut self, store: S, sender: Option<Sender<u32>>) -> Result<()> {
|
||||
rfs::pack(
|
||||
self.meta.clone(),
|
||||
store,
|
||||
&self.docker_tmp_dir.path(),
|
||||
true,
|
||||
sender,
|
||||
)
|
||||
.await
|
||||
.context("failed to pack flist")?;
|
||||
|
||||
log::info!("flist has been created successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn convert<S: Store>(&mut self, store: S, sender: Option<Sender<u32>>) -> Result<()> {
|
||||
self.prepare().await?;
|
||||
self.pack(store, sender).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn extract_image(
|
||||
docker: &Docker,
|
||||
image_name: &str,
|
||||
container_name: &str,
|
||||
docker_tmp_dir_path: &Path,
|
||||
credentials: Option<DockerCredentials>,
|
||||
) -> Result<()> {
|
||||
pull_image(docker, image_name, credentials).await?;
|
||||
create_container(docker, image_name, container_name)
|
||||
.await
|
||||
.context("failed to create docker container")?;
|
||||
export_container(container_name, docker_tmp_dir_path)
|
||||
.context("failed to export docker container")?;
|
||||
container_boot(docker, container_name, docker_tmp_dir_path)
|
||||
.await
|
||||
.context("failed to boot docker container")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn pull_image(
|
||||
docker: &Docker,
|
||||
image_name: &str,
|
||||
credentials: Option<DockerCredentials>,
|
||||
) -> Result<()> {
|
||||
log::info!("pulling docker image {}", image_name);
|
||||
|
||||
let options = Some(CreateImageOptions {
|
||||
from_image: image_name,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let mut image_pull_stream = docker.create_image(options, None, credentials);
|
||||
while let Some(msg) = image_pull_stream.next().await {
|
||||
msg.context("failed to pull docker image")?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn create_container(docker: &Docker, image_name: &str, container_name: &str) -> Result<()> {
|
||||
log::debug!("Inspecting docker image configurations {}", image_name);
|
||||
|
||||
let image = docker
|
||||
.inspect_image(image_name)
|
||||
.await
|
||||
.context("failed to inspect docker image")?;
|
||||
let image_config = image.config.context("failed to get docker image configs")?;
|
||||
|
||||
let mut command = "";
|
||||
if image_config.cmd.is_none() && image_config.entrypoint.is_none() {
|
||||
command = "/bin/sh";
|
||||
}
|
||||
|
||||
log::debug!("Creating a docker container {}", container_name);
|
||||
|
||||
let options = Some(CreateContainerOptions {
|
||||
name: container_name,
|
||||
platform: None,
|
||||
});
|
||||
|
||||
let config = Config {
|
||||
image: Some(image_name),
|
||||
hostname: Some(container_name),
|
||||
cmd: Some(vec![command]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
docker
|
||||
.create_container(options, config)
|
||||
.await
|
||||
.context("failed to create docker temporary container")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn export_container(container_name: &str, docker_tmp_dir_path: &Path) -> Result<()> {
|
||||
log::debug!("Exporting docker container {}", container_name);
|
||||
|
||||
Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(format!(
|
||||
"docker export {} | tar -xpf - -C {}",
|
||||
container_name,
|
||||
docker_tmp_dir_path.display()
|
||||
))
|
||||
.output()
|
||||
.expect("failed to execute export docker container");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn container_boot(
|
||||
docker: &Docker,
|
||||
container_name: &str,
|
||||
docker_tmp_dir_path: &Path,
|
||||
) -> Result<()> {
|
||||
log::debug!(
|
||||
"Inspecting docker container configurations {}",
|
||||
container_name
|
||||
);
|
||||
|
||||
let options = Some(InspectContainerOptions { size: false });
|
||||
let container = docker
|
||||
.inspect_container(container_name, options)
|
||||
.await
|
||||
.context("failed to inspect docker container")?;
|
||||
|
||||
let container_config = container
|
||||
.config
|
||||
.context("failed to get docker container configs")?;
|
||||
|
||||
let command;
|
||||
let args;
|
||||
let mut env: HashMap<String, String> = HashMap::new();
|
||||
let mut cwd = String::from("/");
|
||||
|
||||
let cmd = container_config.cmd.expect("failed to get cmd configs");
|
||||
|
||||
if let Some(entrypoint) = container_config.entrypoint {
|
||||
command = (entrypoint.first().expect("failed to get first entrypoint")).to_string();
|
||||
|
||||
if entrypoint.len() > 1 {
|
||||
let (_, entries) = entrypoint
|
||||
.split_first()
|
||||
.expect("failed to split entrypoint");
|
||||
args = entries.to_vec();
|
||||
} else {
|
||||
args = cmd;
|
||||
}
|
||||
} else {
|
||||
command = (cmd.first().expect("failed to get first cmd")).to_string();
|
||||
let (_, entries) = cmd.split_first().expect("failed to split cmd");
|
||||
args = entries.to_vec();
|
||||
}
|
||||
|
||||
if let Some(envs) = container_config.env {
|
||||
for entry in envs.iter() {
|
||||
if let Some((key, value)) = entry.split_once('=') {
|
||||
env.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref working_dir) = container_config.working_dir {
|
||||
if !working_dir.is_empty() {
|
||||
cwd = working_dir.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
let metadata = json!({
|
||||
"startup": {
|
||||
"entry": {
|
||||
"name": "core.system",
|
||||
"args": {
|
||||
"name": command,
|
||||
"args": args,
|
||||
"env": env,
|
||||
"dir": cwd,
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let toml_metadata: toml::Value = serde_json::from_str(&metadata.to_string())?;
|
||||
|
||||
log::info!(
|
||||
"Creating '.startup.toml' file from container {} contains {}",
|
||||
container_name,
|
||||
toml_metadata.to_string()
|
||||
);
|
||||
|
||||
fs::write(
|
||||
docker_tmp_dir_path.join(".startup.toml"),
|
||||
toml_metadata.to_string(),
|
||||
)
|
||||
.expect("failed to create '.startup.toml' file");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn clean(docker: &Docker, image_name: &str, container_name: &str) -> Result<()> {
|
||||
log::info!("cleaning docker image and container");
|
||||
|
||||
let options = Some(RemoveContainerOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
docker
|
||||
.remove_container(container_name, options)
|
||||
.await
|
||||
.context("failed to remove docker container")?;
|
||||
|
||||
let remove_options = Some(RemoveImageOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
docker
|
||||
.remove_image(image_name, remove_options, None)
|
||||
.await
|
||||
.context("failed to remove docker image")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
115
docker2fl/src/main.rs
Normal file
115
docker2fl/src/main.rs
Normal file
@@ -0,0 +1,115 @@
|
||||
use anyhow::Result;
|
||||
use bollard::auth::DockerCredentials;
|
||||
use clap::{ArgAction, Parser};
|
||||
use rfs::fungi;
|
||||
use rfs::store::parse_router;
|
||||
use tokio::runtime::Builder;
|
||||
use uuid::Uuid;
|
||||
|
||||
mod docker2fl;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(name ="docker2fl", author, version = env!("GIT_VERSION"), about, long_about = None)]
|
||||
struct Options {
|
||||
/// enable debugging logs
|
||||
#[clap(short, long, action=ArgAction::Count)]
|
||||
debug: u8,
|
||||
|
||||
/// store url for rfs in the format [xx-xx=]<url>. the range xx-xx is optional and used for
|
||||
/// sharding. the URL is per store type, please check docs for more information
|
||||
#[clap(short, long, required = true, action=ArgAction::Append)]
|
||||
store: Vec<String>,
|
||||
|
||||
/// name of the docker image to be converted to flist
|
||||
#[clap(short, long, required = true)]
|
||||
image_name: String,
|
||||
|
||||
// docker credentials
|
||||
/// docker hub server username
|
||||
#[clap(long, required = false)]
|
||||
username: Option<String>,
|
||||
|
||||
/// docker hub server password
|
||||
#[clap(long, required = false)]
|
||||
password: Option<String>,
|
||||
|
||||
/// docker hub server auth
|
||||
#[clap(long, required = false)]
|
||||
auth: Option<String>,
|
||||
|
||||
/// docker hub server email
|
||||
#[clap(long, required = false)]
|
||||
email: Option<String>,
|
||||
|
||||
/// docker hub server address
|
||||
#[clap(long, required = false)]
|
||||
server_address: Option<String>,
|
||||
|
||||
/// docker hub server identity token
|
||||
#[clap(long, required = false)]
|
||||
identity_token: Option<String>,
|
||||
|
||||
/// docker hub server registry token
|
||||
#[clap(long, required = false)]
|
||||
registry_token: Option<String>,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let rt = Builder::new_multi_thread()
|
||||
.thread_stack_size(8 * 1024 * 1024)
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap();
|
||||
rt.block_on(run())
|
||||
}
|
||||
|
||||
async fn run() -> Result<()> {
|
||||
let opts = Options::parse();
|
||||
|
||||
simple_logger::SimpleLogger::new()
|
||||
.with_utc_timestamps()
|
||||
.with_level({
|
||||
match opts.debug {
|
||||
0 => log::LevelFilter::Info,
|
||||
1 => log::LevelFilter::Debug,
|
||||
_ => log::LevelFilter::Trace,
|
||||
}
|
||||
})
|
||||
.with_module_level("sqlx", log::Level::Error.to_level_filter())
|
||||
.init()?;
|
||||
|
||||
let mut docker_image = opts.image_name.to_string();
|
||||
if !docker_image.contains(':') {
|
||||
docker_image.push_str(":latest");
|
||||
}
|
||||
|
||||
let credentials = Some(DockerCredentials {
|
||||
username: opts.username,
|
||||
password: opts.password,
|
||||
auth: opts.auth,
|
||||
email: opts.email,
|
||||
serveraddress: opts.server_address,
|
||||
identitytoken: opts.identity_token,
|
||||
registrytoken: opts.registry_token,
|
||||
});
|
||||
|
||||
let fl_name = docker_image.replace([':', '/'], "-") + ".fl";
|
||||
let meta = fungi::Writer::new(&fl_name, true).await?;
|
||||
let store = parse_router(&opts.store).await?;
|
||||
|
||||
let container_name = Uuid::new_v4().to_string();
|
||||
let docker_tmp_dir =
|
||||
tempdir::TempDir::new(&container_name).expect("failed to create tmp directory");
|
||||
|
||||
let mut docker_to_fl =
|
||||
docker2fl::DockerImageToFlist::new(meta, docker_image, credentials, docker_tmp_dir);
|
||||
let res = docker_to_fl.convert(store, None).await;
|
||||
|
||||
// remove the file created with the writer if fl creation failed
|
||||
if res.is_err() {
|
||||
tokio::fs::remove_file(fl_name).await?;
|
||||
return res;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user