diff --git a/Cargo.toml b/Cargo.toml index 1bd23974..546405c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,10 +3,14 @@ resolver = "2" members = [ "crates/*", "examples/*", + "examples/dora/video-capture", + "examples/dora/imgproc", + "examples/dora/video-sink", + "examples/dora/image-utils", "kornia-viz", # "kornia-py", ] -exclude = ["kornia-py"] +exclude = ["kornia-py", "examples/dora"] [workspace.package] authors = ["kornia.org "] diff --git a/examples/dora/.gitignore b/examples/dora/.gitignore new file mode 100644 index 00000000..77d3844f --- /dev/null +++ b/examples/dora/.gitignore @@ -0,0 +1,2 @@ +out +target diff --git a/examples/dora/Cargo.toml b/examples/dora/Cargo.toml new file mode 100644 index 00000000..fee174fb --- /dev/null +++ b/examples/dora/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["video-capture", "video-sink", "imgproc", "image-utils"] diff --git a/examples/dora/README.md b/examples/dora/README.md new file mode 100644 index 00000000..d96fdc6d --- /dev/null +++ b/examples/dora/README.md @@ -0,0 +1,24 @@ +Hello world example for the Kornia Rust library. + +```bash +Usage: hello_world --image-path + +Options: + -i, --image-path + -h, --help Print help +``` + +Example: + +```bash +cargo run -p hello_world -- --image-path path/to/image.jpg +``` + +Output: + +```bash +Hello, world! 🦀 +Loaded Image size: ImageSize { width: 258, height: 195 } + +Goodbyte! +``` diff --git a/examples/dora/dataflow.yml b/examples/dora/dataflow.yml new file mode 100644 index 00000000..1174e9ec --- /dev/null +++ b/examples/dora/dataflow.yml @@ -0,0 +1,37 @@ +nodes: + - id: web-camera + build: cargo build -p dora-video-capture + path: ./target/debug/dora-video-capture + inputs: + tick: dora/timer/millis/10 + outputs: + - frame + env: + SOURCE_TYPE: "webcam" + SOURCE_FPS: 30 + IMAGE_COLS: 640 + IMAGE_ROWS: 480 + - id: rtsp-camera + build: cargo build -p dora-video-capture + path: ./target/debug/dora-video-capture + inputs: + tick: dora/timer/millis/10 + outputs: + - frame + env: + SOURCE_TYPE: "rtsp" + SOURCE_URI: "rtsp://tapo_entrance:123456789@192.168.1.141:554/stream2" + - id: imgproc + build: cargo build -p dora-imgproc + path: ./target/debug/dora-imgproc + inputs: + frame: web-camera/frame + outputs: + - output + - id: video-sink + build: cargo build -p dora-video-sink + path: ./target/debug/dora-video-sink + inputs: + web-camera/frame: web-camera/frame + rtsp-camera/frame: rtsp-camera/frame + imgproc/output: imgproc/output diff --git a/examples/dora/image-utils/Cargo.toml b/examples/dora/image-utils/Cargo.toml new file mode 100644 index 00000000..37dd313a --- /dev/null +++ b/examples/dora/image-utils/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "dora-image-utils" +version = "0.1.0" +edition = "2021" + +[dependencies] +arrow = "54.2.1" +dora-node-api = { version = "0.3", features = ["tracing"] } +eyre = "0.6" +kornia = { version = "0.1.8"} diff --git a/examples/dora/image-utils/src/lib.rs b/examples/dora/image-utils/src/lib.rs new file mode 100644 index 00000000..073fd162 --- /dev/null +++ b/examples/dora/image-utils/src/lib.rs @@ -0,0 +1,89 @@ +use std::collections::BTreeMap; + +use arrow::array::PrimitiveArray; +use arrow::datatypes::UInt8Type; +use dora_node_api::{ArrowData, IntoArrow, Metadata, Parameter}; +use kornia::image::{Image, ImageSize}; + +pub fn image_to_arrow( + image: Image, + metadata: Metadata, +) -> eyre::Result<(BTreeMap, PrimitiveArray)> { + let mut meta_parameters = metadata.parameters; + meta_parameters.insert("cols".to_string(), Parameter::Integer(image.cols() as i64)); + meta_parameters.insert("rows".to_string(), Parameter::Integer(image.rows() as i64)); + + // TODO: avoid to_vec copy + let data = image.as_slice().to_vec().into_arrow(); + + Ok((meta_parameters, data)) +} + +pub fn arrow_to_image(data: ArrowData, metadata: Metadata) -> eyre::Result> { + // SAFETY: we know that the metadata has the "cols" parameter + let img_cols = metadata.parameters.get("cols").unwrap(); + let img_cols: i64 = match img_cols { + Parameter::Integer(i) => *i, + _ => return Err(eyre::eyre!("cols is not an integer")), + }; + + // SAFETY: we know that the metadata has the "rows" parameter + let img_rows = metadata.parameters.get("rows").unwrap(); + let img_rows: i64 = match img_rows { + Parameter::Integer(i) => *i, + _ => return Err(eyre::eyre!("rows is not an integer")), + }; + + let img_data: Vec = TryFrom::try_from(&data)?; + + Ok(Image::new( + ImageSize { + width: img_cols as usize, + height: img_rows as usize, + }, + img_data, + )?) +} + +// fn image_to_union_array(image: Image) -> eyre::Result { +// let mut builder = UnionBuilder::new_dense(); +// builder.append::("cols", image.cols() as i32)?; +// builder.append::("rows", image.rows() as i32)?; +// // TODO: this is not efficient +// for pixel in image.as_slice().iter() { +// builder.append::("data", *pixel)?; +// } +// let union_array = builder.build()?; +// Ok(union_array) +// } + +// fn union_array_to_image(arrow: &arrow::array::UnionArray) -> eyre::Result> { +// let cols = arrow +// .value(0) +// .as_any() +// .downcast_ref::() +// .unwrap() +// .value(0); +// let rows = arrow +// .value(1) +// .as_any() +// .downcast_ref::() +// .unwrap() +// .value(0); +// println!("cols: {}, rows: {}", cols, rows); +// +// let mut data = Vec::with_capacity(cols as usize * rows as usize * 3); +// for i in 0..(cols as usize * rows as usize * 3) { +// data.push( +// arrow +// .value(2 + i) +// .as_any() +// .downcast_ref::() +// .unwrap() +// .value(0), +// ); +// } +// println!("cols: {}, rows: {} len: {}", cols, rows, data.len()); +// let image = Image::new([cols as usize, rows as usize].into(), data)?; +// Ok(image) +// } diff --git a/examples/dora/imgproc/Cargo.toml b/examples/dora/imgproc/Cargo.toml new file mode 100644 index 00000000..9b187a75 --- /dev/null +++ b/examples/dora/imgproc/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "dora-imgproc" +version = "0.1.0" +edition = "2021" + +[dependencies] +dora-node-api = { version = "0.3", features = ["tracing"] } +dora-image-utils = { path = "../image-utils" } +eyre = "0.6" +kornia = { version = "0.1.8"} diff --git a/examples/dora/imgproc/src/main.rs b/examples/dora/imgproc/src/main.rs new file mode 100644 index 00000000..7ef5761c --- /dev/null +++ b/examples/dora/imgproc/src/main.rs @@ -0,0 +1,43 @@ +use dora_image_utils::{arrow_to_image, image_to_arrow}; +use dora_node_api::{self, dora_core::config::DataId, DoraNode, Event}; +use kornia::{image::Image, imgproc}; + +fn main() -> eyre::Result<()> { + let (mut node, mut events) = DoraNode::init_from_env()?; + + let output = DataId::from("output".to_owned()); + + while let Some(event) = events.recv() { + match event { + Event::Input { id, metadata, data } => match id.as_str() { + "frame" => { + // convert the frame to an image + let img = arrow_to_image(data, metadata.clone())?; + + // compute the sobel edge map + let mut out = Image::from_size_val(img.size(), 0f32)?; + imgproc::filter::sobel(&img.cast()?, &mut out, 3)?; + + // TODO: make this more efficient in kornia-image crate + let out_u8 = { + let x = out.map(|x| *x as u8); + Image::new(out.size(), x.into_vec())? + }; + + let (meta_parameters, data) = image_to_arrow(out_u8, metadata)?; + node.send_output(output.clone(), meta_parameters, data)?; + } + other => eprintln!("Ignoring unexpected input `{other}`"), + }, + Event::Stop => { + println!("Received manual stop"); + } + Event::InputClosed { id } => { + println!("Input `{id}` was closed"); + } + other => eprintln!("Received unexpected input: {other:?}"), + } + } + + Ok(()) +} diff --git a/examples/dora/video-capture/Cargo.toml b/examples/dora/video-capture/Cargo.toml new file mode 100644 index 00000000..b8e2d601 --- /dev/null +++ b/examples/dora/video-capture/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "dora-video-capture" +version = "0.1.0" +edition = "2021" + +[dependencies] +dora-node-api = { version = "0.3", features = ["tracing"] } +dora-image-utils = { path = "../image-utils" } +eyre = "0.6" +kornia = { version = "0.1.8", features = ["gstreamer"] } diff --git a/examples/dora/video-capture/src/main.rs b/examples/dora/video-capture/src/main.rs new file mode 100644 index 00000000..a8752790 --- /dev/null +++ b/examples/dora/video-capture/src/main.rs @@ -0,0 +1,60 @@ +use dora_image_utils::image_to_arrow; +use dora_node_api::{self, dora_core::config::DataId, DoraNode, Event}; +use kornia::io::stream::{RTSPCameraConfig, V4L2CameraConfig}; + +fn main() -> eyre::Result<()> { + // parse env variables + let source_type = std::env::var("SOURCE_TYPE")?; + + // create the camera source + let mut camera = match source_type.as_str() { + "webcam" => { + let image_cols = std::env::var("IMAGE_COLS")?.parse::()?; + let image_rows = std::env::var("IMAGE_ROWS")?.parse::()?; + let source_fps = std::env::var("SOURCE_FPS")?.parse::()?; + V4L2CameraConfig::new() + .with_size([image_cols, image_rows].into()) + .with_fps(source_fps) + .build()? + } + "rtsp" => { + let source_uri = std::env::var("SOURCE_URI")?; + RTSPCameraConfig::new().with_url(&source_uri).build()? + } + _ => return Err(eyre::eyre!("Invalid source type: {}", source_type)), + }; + + // start the camera source + camera.start()?; + + let output = DataId::from("frame".to_owned()); + + let (mut node, mut events) = DoraNode::init_from_env()?; + + while let Some(event) = events.recv() { + match event { + Event::Input { + id, + metadata, + data: _, + } => match id.as_str() { + "tick" => { + let Some(frame) = camera.grab()? else { + continue; + }; + + let (meta_parameters, data) = image_to_arrow(frame, metadata)?; + + node.send_output(output.clone(), meta_parameters, data)?; + } + other => eprintln!("Ignoring unexpected input `{other}`"), + }, + Event::Stop => { + camera.close()?; + } + other => eprintln!("Received unexpected input: {other:?}"), + } + } + + Ok(()) +} diff --git a/examples/dora/video-sink/Cargo.toml b/examples/dora/video-sink/Cargo.toml new file mode 100644 index 00000000..52f23971 --- /dev/null +++ b/examples/dora/video-sink/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "dora-video-sink" +version = "0.1.0" +edition = "2021" + +[dependencies] +dora-node-api = { version = "0.3", features = ["tracing"] } +dora-image-utils = { path = "../image-utils" } +eyre = "0.6" +kornia = { version = "0.1.8"} +rerun = "0.22" diff --git a/examples/dora/video-sink/src/main.rs b/examples/dora/video-sink/src/main.rs new file mode 100644 index 00000000..a657667f --- /dev/null +++ b/examples/dora/video-sink/src/main.rs @@ -0,0 +1,44 @@ +use dora_image_utils::arrow_to_image; +use dora_node_api::{DoraNode, Event}; +use kornia::image::Image; + +fn main() -> eyre::Result<()> { + let (mut _node, mut events) = DoraNode::init_from_env()?; + + let rr = rerun::RecordingStreamBuilder::new("Camera Sink").connect_tcp()?; + + while let Some(event) = events.recv() { + match event { + Event::Input { id, metadata, data } => match id.as_str() { + "web-camera/frame" => { + log_image(&rr, "web-camera/frame", &arrow_to_image(data, metadata)?)?; + } + "rtsp-camera/frame" => { + log_image(&rr, "rtsp-camera/frame", &arrow_to_image(data, metadata)?)?; + } + "imgproc/output" => { + log_image(&rr, "imgproc/output", &arrow_to_image(data, metadata)?)?; + } + other => eprintln!("Ignoring unexpected input `{other}`"), + }, + Event::Stop => { + println!("Received manual stop"); + } + Event::InputClosed { id } => { + println!("Input `{id}` was closed"); + } + other => eprintln!("Received unexpected input: {other:?}"), + } + } + + Ok(()) +} + +fn log_image(rr: &rerun::RecordingStream, name: &str, img: &Image) -> eyre::Result<()> { + rr.log_static( + name, + &rerun::Image::from_elements(img.as_slice(), img.size().into(), rerun::ColorModel::RGB), + )?; + + Ok(()) +}