Initial "chunking" code

This analyzes an ostree commit and splits it into chunks
suitable for output to separate layers in an OCI image.

lib/src/cli.rs

@@ -120,6 +120,10 @@ enum ContainerOpts {
         /// Corresponds to the Dockerfile `CMD` instruction.
         #[structopt(long)]
         cmd: Option<Vec<String>>,
+
+        #[structopt(long, hidden = true)]
+        /// Output in multiple blobs
+        ex_chunked: bool,
     },
 
     /// Commands for working with (possibly layered, non-encapsulated) container images.
@@ -231,6 +235,19 @@ struct ImaSignOpts {
     key: String,
 }
 
+/// Experimental options
+#[derive(Debug, StructOpt)]
+enum ExperimentalOpts {
+    /// Print chunking
+    PrintChunks {
+        /// Path to the repository
+        #[structopt(long)]
+        repo: String,
+        /// The ostree ref or commt
+        rev: String,
+    },
+}
+
 /// Toplevel options for extended ostree functionality.
 #[derive(Debug, StructOpt)]
 #[structopt(name = "ostree-ext")]
@@ -243,6 +260,8 @@ enum Opt {
     Container(ContainerOpts),
     /// IMA signatures
     ImaSign(ImaSignOpts),
+    /// Experimental/debug CLI
+    Experimental(ExperimentalOpts),
 }
 
 #[allow(clippy::from_over_into)]
@@ -348,13 +367,17 @@ async fn container_export(
     imgref: &ImageReference,
     labels: BTreeMap<String, String>,
     cmd: Option<Vec<String>>,
+    chunked: bool,
 ) -> Result<()> {
     let repo = &ostree::Repo::open_at(libc::AT_FDCWD, repo, gio::NONE_CANCELLABLE)?;
     let config = Config {
         labels: Some(labels),
         cmd,
     };
-    let opts = Some(Default::default());
+    let opts = Some(crate::container::ExportOpts {
+        chunked,
+        ..Default::default()
+    });
     let pushed = crate::container::encapsulate(repo, rev, &config, opts, imgref).await?;
     println!("{}", pushed);
     Ok(())
@@ -462,6 +485,7 @@ where
                 imgref,
                 labels,
                 cmd,
+                ex_chunked,
             } => {
                 let labels: Result<BTreeMap<_, _>> = labels
                     .into_iter()
@@ -474,7 +498,8 @@ where
                         Ok((k.to_string(), v.to_string()))
                     })
                     .collect();
-                container_export(&repo, &rev, &imgref, labels?, cmd).await
+
+                container_export(&repo, &rev, &imgref, labels?, cmd, ex_chunked).await
             }
             ContainerOpts::Image(opts) => match opts {
                 ContainerImageOpts::List { repo } => {
@@ -527,5 +552,14 @@ where
             },
         },
         Opt::ImaSign(ref opts) => ima_sign(opts),
+        Opt::Experimental(ref opts) => match opts {
+            ExperimentalOpts::PrintChunks { repo, rev } => {
+                let repo = &ostree::Repo::open_at(libc::AT_FDCWD, &repo, gio::NONE_CANCELLABLE)?;
+                let mut chunks = crate::chunking::Chunking::new(repo, rev)?;
+                chunks.auto_chunk(repo)?;
+                crate::chunking::print(&chunks);
+                Ok(())
+            }
+        },
     }
 }

lib/src/container/encapsulate.rs

@@ -2,6 +2,7 @@
 
 use super::ociwriter::OciWriter;
 use super::*;
+use crate::chunking::Chunking;
 use crate::tar as ostree_tar;
 use anyhow::Context;
 use fn_error_context::context;
@@ -40,6 +41,37 @@ fn export_ostree_ref(
     w.complete()
 }
 
+/// Write an ostree commit to an OCI blob
+#[context("Writing ostree root to blob")]
+fn export_chunked(
+    repo: &ostree::Repo,
+    ociw: &mut OciWriter,
+    mut chunking: Chunking,
+    compression: Option<flate2::Compression>,
+) -> Result<()> {
+    let layers: Result<Vec<_>> = chunking
+        .take_chunks()
+        .into_iter()
+        .enumerate()
+        .map(|(i, chunk)| -> Result<_> {
+            let mut w = ociw.create_layer(compression)?;
+            ostree_tar::export_chunk(repo, &chunk, &mut w)
+                .with_context(|| format!("Exporting chunk {}", i))?;
+            let w = w.into_inner()?;
+            Ok((w.complete()?, chunk.name))
+        })
+        .collect();
+    for (layer, name) in layers? {
+        ociw.push_layer(layer, &name);
+    }
+    let mut w = ociw.create_layer(compression)?;
+    ostree_tar::export_final_chunk(repo, &chunking, &mut w)?;
+    let w = w.into_inner()?;
+    let final_layer = w.complete()?;
+    ociw.push_layer(final_layer, "Remaining objects");
+    Ok(())
+}
+
 /// Generate an OCI image from a given ostree root
 #[context("Building oci")]
 fn build_oci(
@@ -60,6 +92,15 @@ fn build_oci(
     let commit_meta = &commit_v.child_value(0);
     let commit_meta = glib::VariantDict::new(Some(commit_meta));
 
+    let chunking = if opts.chunked {
+        // compression = Some(flate2::Compression::none());
+        let mut c = crate::chunking::Chunking::new(repo, commit)?;
+        c.auto_chunk(repo)?;
+        Some(c)
+    } else {
+        None
+    };
+
     if let Some(version) =
         commit_meta.lookup_value("version", Some(glib::VariantTy::new("s").unwrap()))
     {
@@ -84,10 +125,19 @@ fn build_oci(
         flate2::Compression::none()
     };
 
-    let rootfs_blob = export_ostree_ref(repo, commit, &mut writer, Some(compression))?;
     let mut annos = HashMap::new();
     annos.insert(BLOB_OSTREE_ANNOTATION.to_string(), "true".to_string());
-    writer.push_layer_annotated(rootfs_blob, Some(annos));
+
+    if let Some(chunking) = chunking {
+        export_chunked(repo, &mut writer, chunking, Some(compression))?;
+    } else {
+        let rootfs_blob = export_ostree_ref(repo, commit, &mut writer, Some(compression))?;
+        writer.push_layer_annotated(
+            rootfs_blob,
+            Some(annos),
+            &format!("export of commit {}", commit),
+        );
+    }
     writer.complete()?;
 
     Ok(ImageReference {
@@ -167,6 +217,8 @@ async fn build_impl(
 pub struct ExportOpts {
     /// If true, perform gzip compression of the tar layers.
     pub compress: bool,
+    /// Whether or not to generate multiple layers
+    pub chunked: bool,
 }
 
 /// Given an OSTree repository and ref, generate a container image.

lib/src/container/ociwriter.rs

@@ -77,6 +77,7 @@ pub(crate) struct OciWriter<'a> {
     cmd: Option<Vec<String>>,
 
     layers: Vec<(oci_image::Descriptor, String)>,
+    history: Vec<String>,
 }
 
 /// Write a serializable data (JSON) as an OCI blob
@@ -102,6 +103,7 @@ impl<'a> OciWriter<'a> {
             config_annotations: Default::default(),
             manifest_annotations: Default::default(),
             layers: Vec::new(),
+            history: Vec::new(),
             cmd: None,
         })
     }
@@ -123,20 +125,18 @@ impl<'a> OciWriter<'a> {
         Ok(tar::Builder::new(self.create_raw_layer(c)?))
     }
 
-    #[allow(dead_code)]
-    /// Finish all I/O for a layer writer, and add it to the layers in the image.
-    pub(crate) fn finish_and_push_layer(&mut self, w: RawLayerWriter) -> Result<()> {
-        let w = w.complete()?;
-        self.push_layer(w);
-        Ok(())
-    }
-
-    /// Add a layer to the top of the image stack.
-    ///
-    /// The first pushed layer becomes the root.
-    pub(crate) fn push_layer(&mut self, layer: Layer) {
-        let v: Option<HashMap<String, String>> = None;
-        self.push_layer_annotated(layer, v);
+    // #[allow(dead_code)]
+    // /// Finish all I/O for a layer writer, and add it to the layers in the image.
+    // pub(crate) fn finish_and_push_layer(&mut self, w: RawLayerWriter) -> Result<()> {
+    //     let w = w.complete()?;
+    //     self.push_layer(w);
+    //     Ok(())
+    // }
+
+    /// Add a layer to the top of the image stack.  The firsh pushed layer becomes the root.
+    pub(crate) fn push_layer(&mut self, layer: Layer, description: &str) {
+        let annotations: Option<HashMap<String, String>> = None;
+        self.push_layer_annotated(layer, annotations, description);
     }
 
     /// Add a layer to the top of the image stack with optional annotations.
@@ -146,13 +146,15 @@ impl<'a> OciWriter<'a> {
         &mut self,
         layer: Layer,
         annotations: Option<impl Into<HashMap<String, String>>>,
+        description: &str,
     ) {
         let mut builder = layer.descriptor().media_type(MediaType::ImageLayerGzip);
         if let Some(annotations) = annotations {
             builder = builder.annotations(annotations);
         }
         self.layers
             .push((builder.build().unwrap(), layer.uncompressed_sha256));
+        self.history.push(description.to_string());
     }
 
     pub(crate) fn set_cmd(&mut self, e: &[&str]) {
@@ -201,20 +203,22 @@ impl<'a> OciWriter<'a> {
         }
         .build()
         .unwrap();
-        let history = oci_image::HistoryBuilder::default()
-            .created_by(format!(
-                "created by {} {}",
-                env!("CARGO_PKG_NAME"),
-                env!("CARGO_PKG_VERSION")
-            ))
-            .build()
-            .unwrap();
+        let history: Vec<_> = self
+            .history
+            .into_iter()
+            .map(|h| {
+                oci_image::HistoryBuilder::default()
+                    .created_by(h)
+                    .build()
+                    .unwrap()
+            })
+            .collect();
         let config = oci_image::ImageConfigurationBuilder::default()
             .architecture(arch.clone())
             .os(oci_image::Os::Linux)
             .config(ctrconfig)
             .rootfs(rootfs)
-            .history(vec![history])
+            .history(history)
             .build()
             .unwrap();
         let config_blob = write_json_blob(self.dir, &config, MediaType::ImageConfig)?;
@@ -380,7 +384,7 @@ mod tests {
             root_layer.uncompressed_sha256,
             "349438e5faf763e8875b43de4d7101540ef4d865190336c2cc549a11f33f8d7c"
         );
-        w.push_layer(root_layer);
+        w.push_layer(root_layer, "root");
         w.complete()?;
         Ok(())
     }

lib/src/container/store.rs

@@ -139,7 +139,10 @@ pub struct PreparedImport {
 }
 
 // Given a manifest, compute its ostree ref name and cached ostree commit
-fn query_layer(repo: &ostree::Repo, layer: oci_image::Descriptor) -> Result<ManifestLayerState> {
+pub(crate) fn query_layer(
+    repo: &ostree::Repo,
+    layer: oci_image::Descriptor,
+) -> Result<ManifestLayerState> {
     let ostree_ref = ref_for_layer(&layer)?;
     let commit = repo.resolve_rev(&ostree_ref, true)?.map(|s| s.to_string());
     Ok(ManifestLayerState {
@@ -311,9 +314,9 @@ impl LayeredImageImporter {
                     base: Some(base_commit.clone()),
                     selinux: true,
                 };
-                let w =
+                let r =
                     crate::tar::write_tar(&self.repo, blob, layer.ostree_ref.as_str(), Some(opts));
-                let r = super::unencapsulate::join_fetch(w, driver)
+                let r = super::unencapsulate::join_fetch(r, driver)
                     .await
                     .with_context(|| format!("Parsing layer blob {}", layer.digest()))?;
                 layer_commits.push(r.commit);