Dora example

Cargo.toml

@@ -3,10 +3,14 @@ resolver = "2"
 members = [
     "crates/*",
     "examples/*",
+    "examples/dora/video-capture",
+    "examples/dora/imgproc",
+    "examples/dora/video-sink",
+    "examples/dora/image-utils",
     "kornia-viz",
     # "kornia-py",
 ]
-exclude = ["kornia-py"]
+exclude = ["kornia-py", "examples/dora"]
 
 [workspace.package]
 authors = ["kornia.org <[email protected]>"]

examples/dora/.gitignore

@@ -0,0 +1,2 @@
+out
+target

examples/dora/Cargo.toml

@@ -0,0 +1,2 @@
+[workspace]
+members = ["video-capture", "video-sink", "imgproc", "image-utils"]

examples/dora/README.md

@@ -0,0 +1,24 @@
+Hello world example for the Kornia Rust library.
+
+```bash
+Usage: hello_world --image-path <IMAGE_PATH>
+
+Options:
+  -i, --image-path <IMAGE_PATH>
+  -h, --help                     Print help
+```
+
+Example:
+
+```bash
+cargo run -p hello_world -- --image-path path/to/image.jpg
+```
+
+Output:
+
+```bash
+Hello, world! 🦀
+Loaded Image size: ImageSize { width: 258, height: 195 }
+
+Goodbyte!
+```

examples/dora/dataflow.yml

@@ -0,0 +1,37 @@
+nodes:
+  - id: web-camera
+    build: cargo build -p dora-video-capture
+    path: ./target/debug/dora-video-capture
+    inputs:
+      tick: dora/timer/millis/10
+    outputs:
+      - frame
+    env:
+      SOURCE_TYPE: "webcam"
+      SOURCE_FPS: 30
+      IMAGE_COLS: 640
+      IMAGE_ROWS: 480
+  - id: rtsp-camera
+    build: cargo build -p dora-video-capture
+    path: ./target/debug/dora-video-capture
+    inputs:
+      tick: dora/timer/millis/10
+    outputs:
+      - frame
+    env:
+      SOURCE_TYPE: "rtsp"
+      SOURCE_URI: "rtsp://tapo_entrance:[email protected]:554/stream2"
+  - id: imgproc
+    build: cargo build -p dora-imgproc
+    path: ./target/debug/dora-imgproc
+    inputs:
+      frame: web-camera/frame
+    outputs:
+      - output
+  - id: video-sink
+    build: cargo build -p dora-video-sink
+    path: ./target/debug/dora-video-sink
+    inputs:
+      web-camera/frame: web-camera/frame
+      rtsp-camera/frame: rtsp-camera/frame
+      imgproc/output: imgproc/output

examples/dora/image-utils/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "dora-image-utils"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+arrow = "54.2.1"
+dora-node-api = { version = "0.3", features = ["tracing"] }
+eyre = "0.6"
+kornia = { version = "0.1.8"}

examples/dora/image-utils/src/lib.rs

@@ -0,0 +1,89 @@
+use std::collections::BTreeMap;
+
+use arrow::array::PrimitiveArray;
+use arrow::datatypes::UInt8Type;
+use dora_node_api::{ArrowData, IntoArrow, Metadata, Parameter};
+use kornia::image::{Image, ImageSize};
+
+pub fn image_to_arrow(
+    image: Image<u8, 3>,
+    metadata: Metadata,
+) -> eyre::Result<(BTreeMap<String, Parameter>, PrimitiveArray<UInt8Type>)> {
+    let mut meta_parameters = metadata.parameters;
+    meta_parameters.insert("cols".to_string(), Parameter::Integer(image.cols() as i64));
+    meta_parameters.insert("rows".to_string(), Parameter::Integer(image.rows() as i64));
+
+    // TODO: avoid to_vec copy
+    let data = image.as_slice().to_vec().into_arrow();
+
+    Ok((meta_parameters, data))
+}
+
+pub fn arrow_to_image(data: ArrowData, metadata: Metadata) -> eyre::Result<Image<u8, 3>> {
+    // SAFETY: we know that the metadata has the "cols" parameter
+    let img_cols = metadata.parameters.get("cols").unwrap();
+    let img_cols: i64 = match img_cols {
+        Parameter::Integer(i) => *i,
+        _ => return Err(eyre::eyre!("cols is not an integer")),
+    };
+
+    // SAFETY: we know that the metadata has the "rows" parameter
+    let img_rows = metadata.parameters.get("rows").unwrap();
+    let img_rows: i64 = match img_rows {
+        Parameter::Integer(i) => *i,
+        _ => return Err(eyre::eyre!("rows is not an integer")),
+    };
+
+    let img_data: Vec<u8> = TryFrom::try_from(&data)?;
+
+    Ok(Image::new(
+        ImageSize {
+            width: img_cols as usize,
+            height: img_rows as usize,
+        },
+        img_data,
+    )?)
+}
+
+// fn image_to_union_array(image: Image<u8, 3>) -> eyre::Result<UnionArray> {
+//     let mut builder = UnionBuilder::new_dense();
+//     builder.append::<Int32Type>("cols", image.cols() as i32)?;
+//     builder.append::<Int32Type>("rows", image.rows() as i32)?;
+//     // TODO: this is not efficient
+//     for pixel in image.as_slice().iter() {
+//         builder.append::<UInt8Type>("data", *pixel)?;
+//     }
+//     let union_array = builder.build()?;
+//     Ok(union_array)
+// }
+
+// fn union_array_to_image(arrow: &arrow::array::UnionArray) -> eyre::Result<Image<u8, 3>> {
+//     let cols = arrow
+//         .value(0)
+//         .as_any()
+//         .downcast_ref::<arrow::array::Int32Array>()
+//         .unwrap()
+//         .value(0);
+//     let rows = arrow
+//         .value(1)
+//         .as_any()
+//         .downcast_ref::<arrow::array::Int32Array>()
+//         .unwrap()
+//         .value(0);
+//     println!("cols: {}, rows: {}", cols, rows);
+//
+//     let mut data = Vec::with_capacity(cols as usize * rows as usize * 3);
+//     for i in 0..(cols as usize * rows as usize * 3) {
+//         data.push(
+//             arrow
+//                 .value(2 + i)
+//                 .as_any()
+//                 .downcast_ref::<arrow::array::UInt8Array>()
+//                 .unwrap()
+//                 .value(0),
+//         );
+//     }
+//     println!("cols: {}, rows: {} len: {}", cols, rows, data.len());
+//     let image = Image::new([cols as usize, rows as usize].into(), data)?;
+//     Ok(image)
+// }

examples/dora/imgproc/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "dora-imgproc"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+dora-node-api = { version = "0.3", features = ["tracing"] }
+dora-image-utils = { path = "../image-utils" }
+eyre = "0.6"
+kornia = { version = "0.1.8"}

examples/dora/imgproc/src/main.rs

@@ -0,0 +1,43 @@
+use dora_image_utils::{arrow_to_image, image_to_arrow};
+use dora_node_api::{self, dora_core::config::DataId, DoraNode, Event};
+use kornia::{image::Image, imgproc};
+
+fn main() -> eyre::Result<()> {
+    let (mut node, mut events) = DoraNode::init_from_env()?;
+
+    let output = DataId::from("output".to_owned());
+
+    while let Some(event) = events.recv() {
+        match event {
+            Event::Input { id, metadata, data } => match id.as_str() {
+                "frame" => {
+                    // convert the frame to an image
+                    let img = arrow_to_image(data, metadata.clone())?;
+
+                    // compute the sobel edge map
+                    let mut out = Image::from_size_val(img.size(), 0f32)?;
+                    imgproc::filter::sobel(&img.cast()?, &mut out, 3)?;
+
+                    // TODO: make this more efficient in kornia-image crate
+                    let out_u8 = {
+                        let x = out.map(|x| *x as u8);
+                        Image::new(out.size(), x.into_vec())?
+                    };
+
+                    let (meta_parameters, data) = image_to_arrow(out_u8, metadata)?;
+                    node.send_output(output.clone(), meta_parameters, data)?;
+                }
+                other => eprintln!("Ignoring unexpected input `{other}`"),
+            },
+            Event::Stop => {
+                println!("Received manual stop");
+            }
+            Event::InputClosed { id } => {
+                println!("Input `{id}` was closed");
+            }
+            other => eprintln!("Received unexpected input: {other:?}"),
+        }
+    }
+
+    Ok(())
+}

examples/dora/video-capture/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "dora-video-capture"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+dora-node-api = { version = "0.3", features = ["tracing"] }
+dora-image-utils = { path = "../image-utils" }
+eyre = "0.6"
+kornia = { version = "0.1.8", features = ["gstreamer"] }

examples/dora/video-capture/src/main.rs

@@ -0,0 +1,60 @@
+use dora_image_utils::image_to_arrow;
+use dora_node_api::{self, dora_core::config::DataId, DoraNode, Event};
+use kornia::io::stream::{RTSPCameraConfig, V4L2CameraConfig};
+
+fn main() -> eyre::Result<()> {
+    // parse env variables
+    let source_type = std::env::var("SOURCE_TYPE")?;
+
+    // create the camera source
+    let mut camera = match source_type.as_str() {
+        "webcam" => {
+            let image_cols = std::env::var("IMAGE_COLS")?.parse::<usize>()?;
+            let image_rows = std::env::var("IMAGE_ROWS")?.parse::<usize>()?;
+            let source_fps = std::env::var("SOURCE_FPS")?.parse::<u32>()?;
+            V4L2CameraConfig::new()
+                .with_size([image_cols, image_rows].into())
+                .with_fps(source_fps)
+                .build()?
+        }
+        "rtsp" => {
+            let source_uri = std::env::var("SOURCE_URI")?;
+            RTSPCameraConfig::new().with_url(&source_uri).build()?
+        }
+        _ => return Err(eyre::eyre!("Invalid source type: {}", source_type)),
+    };
+
+    // start the camera source
+    camera.start()?;
+
+    let output = DataId::from("frame".to_owned());
+
+    let (mut node, mut events) = DoraNode::init_from_env()?;
+
+    while let Some(event) = events.recv() {
+        match event {
+            Event::Input {
+                id,
+                metadata,
+                data: _,
+            } => match id.as_str() {
+                "tick" => {
+                    let Some(frame) = camera.grab()? else {
+                        continue;
+                    };
+
+                    let (meta_parameters, data) = image_to_arrow(frame, metadata)?;
+
+                    node.send_output(output.clone(), meta_parameters, data)?;
+                }
+                other => eprintln!("Ignoring unexpected input `{other}`"),
+            },
+            Event::Stop => {
+                camera.close()?;
+            }
+            other => eprintln!("Received unexpected input: {other:?}"),
+        }
+    }
+
+    Ok(())
+}

examples/dora/video-sink/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "dora-video-sink"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+dora-node-api = { version = "0.3", features = ["tracing"] }
+dora-image-utils = { path = "../image-utils" }
+eyre = "0.6"
+kornia = { version = "0.1.8"}
+rerun = "0.22"

examples/dora/video-sink/src/main.rs

@@ -0,0 +1,44 @@
+use dora_image_utils::arrow_to_image;
+use dora_node_api::{DoraNode, Event};
+use kornia::image::Image;
+
+fn main() -> eyre::Result<()> {
+    let (mut _node, mut events) = DoraNode::init_from_env()?;
+
+    let rr = rerun::RecordingStreamBuilder::new("Camera Sink").connect_tcp()?;
+
+    while let Some(event) = events.recv() {
+        match event {
+            Event::Input { id, metadata, data } => match id.as_str() {
+                "web-camera/frame" => {
+                    log_image(&rr, "web-camera/frame", &arrow_to_image(data, metadata)?)?;
+                }
+                "rtsp-camera/frame" => {
+                    log_image(&rr, "rtsp-camera/frame", &arrow_to_image(data, metadata)?)?;
+                }
+                "imgproc/output" => {
+                    log_image(&rr, "imgproc/output", &arrow_to_image(data, metadata)?)?;
+                }
+                other => eprintln!("Ignoring unexpected input `{other}`"),
+            },
+            Event::Stop => {
+                println!("Received manual stop");
+            }
+            Event::InputClosed { id } => {
+                println!("Input `{id}` was closed");
+            }
+            other => eprintln!("Received unexpected input: {other:?}"),
+        }
+    }
+
+    Ok(())
+}
+
+fn log_image(rr: &rerun::RecordingStream, name: &str, img: &Image<u8, 3>) -> eyre::Result<()> {
+    rr.log_static(
+        name,
+        &rerun::Image::from_elements(img.as_slice(), img.size().into(), rerun::ColorModel::RGB),
+    )?;
+
+    Ok(())
+}