diff --git a/bin/oay/Cargo.toml b/bin/oay/Cargo.toml index 378b00d66e3a..b0a962417fbf 100644 --- a/bin/oay/Cargo.toml +++ b/bin/oay/Cargo.toml @@ -29,6 +29,18 @@ repository.workspace = true rust-version.workspace = true version.workspace = true +[features] +default = [ + "frontends-webdav", + "frontends-s3" +] + +frontends-webdav = [ + "dep:dav-server", + "dep:bytes" +] +frontends-s3 = [] + [dependencies] anyhow = "1" axum = "0.6" @@ -52,3 +64,5 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } url = "2.3.1" uuid = { version = "1", features = ["v4", "fast-rng"] } +dav-server = { version = "0.5.5", optional = true } +bytes = { version = "1.4.0", optional = true } \ No newline at end of file diff --git a/bin/oay/src/bin/oay.rs b/bin/oay/src/bin/oay.rs index 93d846fe1acb..a2324aa5fdd4 100644 --- a/bin/oay/src/bin/oay.rs +++ b/bin/oay/src/bin/oay.rs @@ -21,7 +21,9 @@ use std::sync::Arc; use anyhow::Context; use anyhow::Result; use oay::services::S3Service; +use oay::services::WebdavService; use oay::Config; +use opendal::services::Fs; use opendal::Operator; use opendal::Scheme; use tracing_subscriber::fmt; @@ -30,6 +32,11 @@ use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() -> Result<()> { + let _ = s3().await; + webdav().await +} + +async fn s3() -> Result<()> { tracing_subscriber::registry() .with(fmt::layer().pretty()) .with(EnvFilter::from_default_env()) @@ -46,3 +53,30 @@ async fn main() -> Result<()> { Ok(()) } + +async fn webdav() -> Result<()> { + let cfg: Config = Config { + backend: oay::BackendConfig { + typ: "fs".to_string(), + ..Default::default() + }, + frontends: oay::FrontendsConfig { + webdav: oay::WebdavConfig { + enable: true, + addr: "127.0.0.1:3000".to_string(), + }, + ..Default::default() + }, + }; + + let mut builder = Fs::default(); + builder.root("/tmp"); + + let op = Operator::new(builder)?.finish(); + + let webdav = WebdavService::new(Arc::new(cfg), op); + + webdav.serve().await?; + + Ok(()) +} diff --git a/bin/oay/src/bin/webdav.rs b/bin/oay/src/bin/webdav.rs new file mode 100644 index 000000000000..3250dffcba79 --- /dev/null +++ b/bin/oay/src/bin/webdav.rs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use anyhow::Result; + +use oay::services::WebdavService; +use oay::Config; +use opendal::services::Fs; + +use opendal::Operator; +use tracing_subscriber::fmt; +use tracing_subscriber::prelude::*; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::registry() + .with(fmt::layer().pretty()) + .with(EnvFilter::from_default_env()) + .init(); + + let cfg: Config = Config { + backend: oay::BackendConfig { + typ: "fs".to_string(), + ..Default::default() + }, + frontends: oay::FrontendsConfig { + webdav: oay::WebdavConfig { + enable: true, + addr: "127.0.0.1:3000".to_string(), + }, + ..Default::default() + }, + }; + + let mut builder = Fs::default(); + builder.root("/tmp"); + + let op = Operator::new(builder)?.finish(); + + let webdav = WebdavService::new(Arc::new(cfg), op); + + webdav.serve().await?; + + Ok(()) +} diff --git a/bin/oay/src/config.rs b/bin/oay/src/config.rs index f306ed9fac21..c38a270b96bf 100644 --- a/bin/oay/src/config.rs +++ b/bin/oay/src/config.rs @@ -26,7 +26,7 @@ pub struct Config { pub frontends: FrontendsConfig, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Default)] pub struct BackendConfig { #[serde(rename = "type")] pub typ: String, @@ -34,13 +34,20 @@ pub struct BackendConfig { pub map: HashMap, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Default)] pub struct FrontendsConfig { pub s3: S3Config, + pub webdav: WebdavConfig, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Default)] pub struct S3Config { pub enable: bool, pub addr: String, } + +#[derive(Serialize, Deserialize, Default)] +pub struct WebdavConfig { + pub enable: bool, + pub addr: String, +} diff --git a/bin/oay/src/services/mod.rs b/bin/oay/src/services/mod.rs index 001951bef0b9..e54b00dcc06c 100644 --- a/bin/oay/src/services/mod.rs +++ b/bin/oay/src/services/mod.rs @@ -15,5 +15,12 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "frontends-s3")] mod s3; +#[cfg(feature = "frontends-s3")] pub use s3::S3Service; + +#[cfg(feature = "frontends-webdav")] +mod webdav; +#[cfg(feature = "frontends-webdav")] +pub use webdav::WebdavService; diff --git a/bin/oay/src/services/webdav/mod.rs b/bin/oay/src/services/webdav/mod.rs new file mode 100644 index 000000000000..e432889a8cb5 --- /dev/null +++ b/bin/oay/src/services/webdav/mod.rs @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod service; +mod webdav_dir_entry; +mod webdav_file; +mod webdav_metadata; +pub mod webdavfs; +pub use service::*; diff --git a/bin/oay/src/services/webdav/service.rs b/bin/oay/src/services/webdav/service.rs new file mode 100644 index 000000000000..dc3045f9683f --- /dev/null +++ b/bin/oay/src/services/webdav/service.rs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::convert::Infallible; +use std::sync::Arc; + +use axum::body::Body; +use axum::http::Request; +use axum::routing::any_service; +use axum::Router; +use dav_server::DavHandler; +use opendal::Operator; + +use crate::Config; + +use super::webdavfs::WebdavFs; + +pub struct WebdavService { + cfg: Arc, + webdavfs: Box, +} + +impl WebdavService { + pub fn new(cfg: Arc, op: Operator) -> Self { + Self { + cfg, + webdavfs: WebdavFs::new(op), + } + } + + pub async fn serve(&self) -> anyhow::Result<()> { + let webdav_cfg = &self.cfg.frontends.webdav; + + let webdav_handler = DavHandler::builder() + .filesystem(self.webdavfs.clone()) + .build_handler(); + + let webdav_service = tower::service_fn(move |req: Request| { + let webdav_server = webdav_handler.clone(); + async move { Ok::<_, Infallible>(webdav_server.handle(req).await) } + }); + + let app = Router::new().route("/*path", any_service(webdav_service)); + + axum::Server::bind(&webdav_cfg.addr.parse().unwrap()) + .serve(app.into_make_service()) + .await?; + + Ok(()) + } +} diff --git a/bin/oay/src/services/webdav/webdav_dir_entry.rs b/bin/oay/src/services/webdav/webdav_dir_entry.rs new file mode 100644 index 000000000000..e782ef2e7928 --- /dev/null +++ b/bin/oay/src/services/webdav/webdav_dir_entry.rs @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use dav_server::fs::DavDirEntry; + +struct WebDAVDirEntry {} + +impl DavDirEntry for WebDAVDirEntry { + fn name(&self) -> Vec { + todo!() + } + + fn metadata(&self) -> dav_server::fs::FsFuture> { + todo!() + } +} diff --git a/bin/oay/src/services/webdav/webdav_file.rs b/bin/oay/src/services/webdav/webdav_file.rs new file mode 100644 index 000000000000..ed9ac1ba6507 --- /dev/null +++ b/bin/oay/src/services/webdav/webdav_file.rs @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::io::SeekFrom; + +use dav_server::{ + davpath::DavPath, + fs::{DavFile, DavMetaData, FsFuture, OpenOptions}, +}; +use futures::FutureExt; +use opendal::Operator; + +use super::webdav_metadata::WebdavMetaData; + +#[derive(Debug)] +pub struct WebdavFile { + pub op: Operator, + pub path: DavPath, + pub options: OpenOptions, +} + +impl DavFile for WebdavFile { + fn read_bytes(&mut self, count: usize) -> FsFuture { + async move { + let file_path = self.path.as_rel_ospath(); + let content = self + .op + .range_read(file_path.to_str().unwrap(), 0..count as u64) + .await + .unwrap(); + //error handle ? + Ok(bytes::Bytes::from(content)) + } + .boxed() + } + + fn metadata(&mut self) -> FsFuture> { + async move { + let opendal_metadata = self + .op + .stat(self.path.as_rel_ospath().to_str().unwrap()) + .await + .unwrap(); + Ok(Box::new(WebdavMetaData::new(opendal_metadata)) as Box) + } + .boxed() + } + + fn write_buf(&mut self, _buf: Box) -> FsFuture<()> { + todo!() + } + + fn write_bytes(&mut self, _buf: bytes::Bytes) -> FsFuture<()> { + todo!() + } + + fn seek(&mut self, _pos: SeekFrom) -> FsFuture { + todo!() + } + + fn flush(&mut self) -> FsFuture<()> { + todo!() + } +} diff --git a/bin/oay/src/services/webdav/webdav_metadata.rs b/bin/oay/src/services/webdav/webdav_metadata.rs new file mode 100644 index 000000000000..439bb40f8cd9 --- /dev/null +++ b/bin/oay/src/services/webdav/webdav_metadata.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use dav_server::fs::DavMetaData; +use opendal::Metadata; + +#[derive(Debug, Clone)] +pub struct WebdavMetaData { + metadata: Metadata, +} + +impl WebdavMetaData { + pub fn new(metadata: Metadata) -> Self { + WebdavMetaData { metadata } + } +} + +impl DavMetaData for WebdavMetaData { + fn len(&self) -> u64 { + self.metadata.content_length() + } + + fn modified(&self) -> dav_server::fs::FsResult { + Ok(self.metadata.last_modified().unwrap().into()) + } + + fn is_dir(&self) -> bool { + self.metadata.is_dir() + } +} diff --git a/bin/oay/src/services/webdav/webdavfs.rs b/bin/oay/src/services/webdav/webdavfs.rs new file mode 100644 index 000000000000..7c5709aaff99 --- /dev/null +++ b/bin/oay/src/services/webdav/webdavfs.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use dav_server::fs::{DavFile, DavFileSystem, DavMetaData}; +use futures::FutureExt; +use opendal::Operator; + +use super::{webdav_file::WebdavFile, webdav_metadata::WebdavMetaData}; + +#[derive(Clone)] +pub struct WebdavFs { + pub op: Operator, +} + +impl DavFileSystem for WebdavFs { + fn open<'a>( + &'a self, + path: &'a dav_server::davpath::DavPath, + options: dav_server::fs::OpenOptions, + ) -> dav_server::fs::FsFuture> { + async move { + let file = WebdavFile { + op: self.op.clone(), + path: path.clone(), + options, + }; + Ok(Box::new(file) as Box) + } + .boxed() + } + + fn read_dir<'a>( + &'a self, + _path: &'a dav_server::davpath::DavPath, + _meta: dav_server::fs::ReadDirMeta, + ) -> dav_server::fs::FsFuture>> + { + todo!() + } + + fn metadata<'a>( + &'a self, + path: &'a dav_server::davpath::DavPath, + ) -> dav_server::fs::FsFuture> { + async move { + let opendal_metadata = self + .op + .stat(path.as_rel_ospath().to_str().unwrap()) + .await + .unwrap(); + Ok(Box::new(WebdavMetaData::new(opendal_metadata)) as Box) + } + .boxed() + } +} + +impl WebdavFs { + pub fn new(op: Operator) -> Box { + Box::new(WebdavFs { op }) + } +}