Fix martin-ui packaging (#1699)

Run `npm install` and javascript packaging inside the `OUT_DIR`


```
error: failed to verify package tarball

Caused by:
  Source directory was modified by build.rs during cargo publish. Build scripts should not modify anything outside of OUT_DIR.
  Added: .../martin/target/package/martin-0.15.0/martin-ui/dist
  	.../martin/target/package/martin-0.15.0/martin-ui/dist/_
  	.../martin/target/package/martin-0.15.0/martin-ui/dist/_/assets
        ...
```

Fixes #1667
See prior partial fix in #1668

---------

Co-authored-by: Frank Elsinga <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Frank Elsinga <[email protected]>

.github/workflows/release.yml

@@ -0,0 +1,32 @@
+name: Packaging and release
+
+# For now this is only testing that packaging will be possible in the future
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - '**.md'
+      - 'demo/**'
+      - 'docs/**'
+  pull_request:
+    branches: [ main ]
+    paths-ignore:
+      - '**.md'
+      - 'demo/**'
+      - 'docs/**'
+  workflow_dispatch:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+      - name: Install nightly toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly
+      - name: Test packaging
+        run: cargo publish --workspace -Z package-workspace --dry-run

Cargo.toml

@@ -92,6 +92,7 @@ tilejson = "0.4"
 tokio = { version = "1", features = ["macros"] }
 tokio-postgres-rustls = "0.13"
 url = "2.5"
+walkdir = "2.5.0"
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 
 [profile.dev.package]

martin/Cargo.toml

@@ -56,7 +56,7 @@ harness = false
 
 [features]
 default = ["webui", "fonts", "lambda", "mbtiles", "pmtiles", "cog", "postgres", "sprites"]
-webui = ["dep:actix-web-static-files", "dep:static-files"]
+webui = ["dep:actix-web-static-files", "dep:static-files", "dep:walkdir"]
 fonts = ["dep:bit-set", "dep:pbf_font_tools"]
 lambda = ["dep:lambda-web"]
 mbtiles = ["dep:mbtiles"]
@@ -114,6 +114,7 @@ tokio-postgres-rustls = { workspace = true, optional = true }
 url.workspace = true
 
 [build-dependencies]
+walkdir = { workspace = true, optional = true }
 static-files = { workspace = true, optional = true }
 
 [dev-dependencies]

martin/build.rs

@@ -1,13 +1,107 @@
-fn main() -> std::io::Result<()> {
-    #[cfg(feature = "webui")]
-    {
-        static_files::NpmBuild::new("martin-ui")
-            .install()?
-            .run("build")?
-            .target("martin-ui/dist")
-            .change_detection()
-            .to_resource_dir()
-            .build()?;
+#[cfg(feature = "webui")]
+use std::fs;
+#[cfg(feature = "webui")]
+use std::path::Path;
+
+#[cfg(feature = "webui")]
+fn copy_file_tree(src: &Path, dst: &Path, exclude_dirs: &[&str]) {
+    assert!(
+        src.is_dir(),
+        "source for the copy operation is not an existing directory"
+    );
+    let _ = fs::remove_dir_all(dst); // ignore if dir does not exist
+    fs::create_dir_all(dst).unwrap_or_else(|e| {
+        panic!(
+            "failed to create destination directory {}: {e}",
+            dst.display()
+        )
+    });
+    let excludes = exclude_dirs.iter().map(|v| src.join(v)).collect::<Vec<_>>();
+
+    let mut it = walkdir::WalkDir::new(src).follow_links(true).into_iter();
+    while let Some(entry) = it.next() {
+        let entry = entry.expect("failed to read directory entry");
+        if excludes.iter().any(|v| v == entry.path()) {
+            it.skip_current_dir();
+            continue;
+        }
+
+        // Get the relative path of the entry
+        let dst_path = dst.join(
+            entry
+                .path()
+                .strip_prefix(src)
+                .expect("path is not a prefix of the source directory"),
+        );
+
+        if entry.file_type().is_dir() {
+            fs::create_dir_all(&dst_path).unwrap_or_else(|e| {
+                panic!(
+                    "failed to create destination directory {}: {e}",
+                    dst_path.display()
+                )
+            });
+        } else {
+            fs::copy(entry.path(), &dst_path).unwrap_or_else(|e| {
+                panic!(
+                    "failed to copy file {} to {}: {e}",
+                    entry.path().display(),
+                    dst_path.display()
+                )
+            });
+        }
     }
-    Ok(())
+}
+
+#[cfg(feature = "webui")]
+fn webui() {
+    // rust requires that all changes are done in OUT_DIR.
+    //
+    // We thus need to
+    // - move the frontend code to the OUT_DIR
+    // - install npm dependencies
+    // - build the frontend
+    let martin_ui_dir = std::env::current_dir()
+        .expect("Unable to get current dir")
+        .join("martin-ui");
+    assert!(martin_ui_dir.is_dir(), "martin-ui directory does not exist");
+
+    let out_martin_ui_dir = std::env::var("OUT_DIR")
+        .expect("OUT_DIR environment variable is not set")
+        .parse::<std::path::PathBuf>()
+        .expect("OUT_DIR environment variable is not a valid path")
+        .join("martin-ui");
+
+    copy_file_tree(
+        &martin_ui_dir,
+        &out_martin_ui_dir,
+        &["dist", "node_modules"],
+    );
+
+    println!("installing and building in {out_martin_ui_dir:?}");
+    static_files::NpmBuild::new(&out_martin_ui_dir)
+        .install()
+        .expect("npm install failed")
+        .run("build")
+        .expect("npm run build failed")
+        .target(out_martin_ui_dir.join("dist"))
+        .to_resource_dir()
+        .build()
+        .expect("failed to build webui npm dir");
+
+    let target_to_keep = martin_ui_dir.join("dist");
+    assert!(
+        !target_to_keep.exists() || target_to_keep.is_dir(),
+        "the martin-ui/dist must either not exist or have been produced by previous builds"
+    );
+
+    // TODO: we may need to move index.html one level down per change_detection() docs
+    static_files::NpmBuild::new(martin_ui_dir)
+        .target(&target_to_keep)
+        .change_detection();
+}
+
+fn main() {
+    #[cfg(feature = "webui")]
+    webui();
 }

martin/src/cog/config.rs

@@ -1,13 +1,13 @@
-use std::{fmt::Debug, path::PathBuf};
+use std::fmt::Debug;
+use std::path::PathBuf;
 
 use serde::{Deserialize, Serialize};
 use url::Url;
 
-use crate::config::UnrecognizedValues;
-use crate::file_config::{ConfigExtras, SourceConfigExtras};
-use crate::{file_config::FileResult, Source};
-
 use super::source::CogSource;
+use crate::config::UnrecognizedValues;
+use crate::file_config::{ConfigExtras, FileResult, SourceConfigExtras};
+use crate::Source;
 
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct CogConfig {

martin/src/cog/source.rs

@@ -1,22 +1,20 @@
 use std::collections::HashMap;
+use std::fmt::Debug;
 use std::fs::File;
-use std::path::Path;
+use std::io::BufWriter;
+use std::path::{Path, PathBuf};
 use std::vec;
-use std::{fmt::Debug, path::PathBuf};
 
+use async_trait::async_trait;
 use log::warn;
-use std::io::BufWriter;
+use martin_tile_utils::{Format, TileCoord, TileInfo};
 use tiff::decoder::{ChunkType, Decoder, DecodingResult};
 use tiff::tags::Tag::{self, GdalNodata};
-
-use async_trait::async_trait;
-use martin_tile_utils::{Format, TileCoord, TileInfo};
 use tilejson::{tilejson, TileJSON};
 
-use crate::file_config::FileError;
-use crate::{file_config::FileResult, MartinResult, Source, TileData, UrlQuery};
-
 use super::CogError;
+use crate::file_config::{FileError, FileResult};
+use crate::{MartinResult, Source, TileData, UrlQuery};
 
 #[derive(Clone, Debug)]
 struct Meta {
@@ -384,9 +382,10 @@ fn get_images_ifd(decoder: &mut Decoder<File>, path: &Path) -> Vec<usize> {
 
 #[cfg(test)]
 mod tests {
+    use std::path::PathBuf;
+
     use martin_tile_utils::TileCoord;
     use rstest::rstest;
-    use std::path::PathBuf;
 
     use crate::cog::source::get_tile_idx;
 

martin/src/pg/config.rs

@@ -12,8 +12,7 @@ use crate::pg::builder::PgBuilder;
 use crate::pg::config_function::FuncInfoSources;
 use crate::pg::config_table::TableInfoSources;
 use crate::pg::utils::on_slow;
-use crate::pg::PgError;
-use crate::pg::PgResult;
+use crate::pg::{PgError, PgResult};
 use crate::source::TileInfoSources;
 use crate::utils::{IdResolver, OptBoolObj, OptOneMany};
 use crate::MartinResult;

martin/src/pg/pool.rs

@@ -190,13 +190,14 @@ SELECT (regexp_matches(
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use deadpool_postgres::tokio_postgres::Config;
     use postgres::NoTls;
     use testcontainers_modules::postgres::Postgres;
     use testcontainers_modules::testcontainers::runners::AsyncRunner as _;
     use testcontainers_modules::testcontainers::ImageExt as _;
 
+    use super::*;
+
     #[tokio::test]
     async fn parse_version() -> anyhow::Result<()> {
         let node = Postgres::default()

martin/src/sprites/mod.rs

@@ -1,3 +1,7 @@
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::path::PathBuf;
+
 use dashmap::{DashMap, Entry};
 use futures::future::try_join_all;
 use log::{info, warn};
@@ -6,9 +10,6 @@ use spreet::resvg::usvg::{Error as ResvgError, Options, Tree, TreeParsing};
 use spreet::{
     get_svg_input_paths, sprite_name, SpreetError, Sprite, Spritesheet, SpritesheetBuilder,
 };
-use std::collections::BTreeMap;
-use std::fmt::Debug;
-use std::path::PathBuf;
 use tokio::io::AsyncReadExt;
 
 use self::SpriteError::{SpriteInstError, SpriteParsingError, SpriteProcessingError};

martin/src/srv/server.rs

@@ -28,6 +28,7 @@ use crate::MartinResult;
 #[cfg(feature = "webui")]
 mod webui {
     #![allow(clippy::unreadable_literal)]
+    #![allow(clippy::too_many_lines)]
     #![allow(clippy::wildcard_imports)]
     include!(concat!(env!("OUT_DIR"), "/generated.rs"));
 }