Merge pull request #2149 from nulib/deploy/staging

Deploy v 1.6.0 to production

lib/meadow/data/preservation_check_writer.ex

@@ -110,8 +110,8 @@ defmodule Meadow.Data.PreservationCheckWriter do
         file_set.id,
         file_set.metadata.label,
         file_set.role.id,
-        Map.get(file_set.metadata.digests, "sha256"),
-        Map.get(file_set.metadata.digests, "sha1"),
+        get_if_map(result.digests, "sha256"),
+        get_if_map(result.digests, "sha1"),
         file_set.metadata.location,
         Map.fetch!(result, :preservation),
         FileSets.pyramid_uri_for(file_set),
@@ -120,19 +120,33 @@ defmodule Meadow.Data.PreservationCheckWriter do
     end
   end
 
+  defp get_if_map(map, key) when is_map(map) do
+    Map.get(map, key, "MISSING")
+  end
+
+  defp get_if_map(_, _), do: "MISSING"
+
   defp check_files(file_set) do
     %{
+      :digests => file_set.metadata |> Map.get(:digests),
       :preservation => validate_preservation_file(file_set.metadata.location),
       :pyramid => validate_pyramid_present(file_set)
     }
   end
 
-  defp record_invalid_file_set(%{:preservation => false, :pyramid => _}, cache_key),
+  defp record_invalid_file_set(%{preservation: false}, cache_key),
     do: record_invalid_file_set(cache_key)
 
-  defp record_invalid_file_set(%{:pyramid => false, :preservation => _}, cache_key),
+  defp record_invalid_file_set(%{pyramid: false}, cache_key),
     do: record_invalid_file_set(cache_key)
 
+  defp record_invalid_file_set(%{digests: digests}, cache_key) do
+    case digests do
+      %{"sha256" => <<_sha256::binary-size(64)>>, "sha1" => <<_sha1::binary-size(40)>>} -> :noop
+      _ -> record_invalid_file_set(cache_key)
+    end
+  end
+
   defp record_invalid_file_set(_, _cache_key), do: :noop
 
   defp record_invalid_file_set(cache_key) do

mix.exs

@@ -1,7 +1,7 @@
 defmodule Meadow.MixProject do
   use Mix.Project
 
-  @app_version "1.5.2"
+  @app_version "1.6.0"
 
   def project do
     [

priv/elasticsearch/meadow.json

@@ -60,7 +60,10 @@
         {
           "ids": {
             "path_match": "*.id",
-            "mapping": { "type": "keyword" }
+            "mapping": { 
+              "type": "keyword",
+              "copy_to": ["all_controlled_terms"] 
+            }
           }
         },
         {
@@ -97,7 +100,7 @@
               "fields": {
                 "keyword": { "type": "keyword" }
               },
-              "copy_to": ["full_text"]
+              "copy_to": ["full_text", "all_controlled_labels"]
             }
           }
         },

test/meadow/data/preservation_check_writer_test.exs

@@ -64,22 +64,21 @@ defmodule Meadow.Data.PreservationCheckWriterTest do
   end
 
   describe "generate_report/1" do
-    setup do
+    setup %{file_set_1: file_set_1, file_set_2: file_set_2} do
       on_exit(fn ->
         delete_object(@preservation_check_bucket, @report_filename)
+        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_1.id))
+        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_2.id))
       end)
+
+      :ok
     end
 
     @describetag s3: [@preservation_fixture]
     test "generates and uploads a preservation check report", %{
       file_set_1: file_set_1,
       file_set_2: file_set_2
     } do
-      on_exit(fn ->
-        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_1.id))
-        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_2.id))
-      end)
-
       CreatePyramidTiff.process(%{file_set_id: file_set_1.id}, %{})
       CreatePyramidTiff.process(%{file_set_id: file_set_2.id}, %{})
 
@@ -94,11 +93,6 @@ defmodule Meadow.Data.PreservationCheckWriterTest do
       file_set_1: file_set_1,
       file_set_2: file_set_2
     } do
-      on_exit(fn ->
-        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_1.id))
-        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_2.id))
-      end)
-
       CreatePyramidTiff.process(%{file_set_id: file_set_1.id}, %{})
       CreatePyramidTiff.process(%{file_set_id: file_set_2.id}, %{})
 
@@ -120,11 +114,6 @@ defmodule Meadow.Data.PreservationCheckWriterTest do
       file_set_1: file_set_1,
       file_set_2: file_set_2
     } do
-      on_exit(fn ->
-        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_1.id))
-        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_2.id))
-      end)
-
       CreatePyramidTiff.process(%{file_set_id: file_set_1.id}, %{})
       CreatePyramidTiff.process(%{file_set_id: file_set_2.id}, %{})
 
@@ -142,20 +131,29 @@ defmodule Meadow.Data.PreservationCheckWriterTest do
 
     @describetag s3: [@preservation_fixture]
     test "records an error if pyramid file not found in expected location", %{
-      file_set_1: file_set_1,
-      file_set_2: file_set_2
+      file_set_1: file_set_1
     } do
-      on_exit(fn ->
-        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_1.id))
-        delete_object(@pyramid_bucket, Pairtree.pyramid_path(file_set_2.id))
-      end)
-
       CreatePyramidTiff.process(%{file_set_id: file_set_1.id}, %{})
 
       assert {:ok, "s3://test-preservation-checks/pres_check.csv", 1} =
                PreservationCheckWriter.generate_report(@report_filename)
 
       assert object_exists?(@preservation_check_bucket, @report_filename)
     end
+
+    @describetag s3: [@preservation_fixture]
+    test "records an error if file set digests are missing", %{
+      file_set_1: file_set_1,
+      file_set_2: file_set_2
+    } do
+      FileSets.update_file_set(file_set_1, %{metadata: %{digests: nil}})
+      FileSets.update_file_set(file_set_2, %{metadata: %{digests: %{"sha256" => "badsha"}}})
+
+      CreatePyramidTiff.process(%{file_set_id: file_set_1.id}, %{})
+      CreatePyramidTiff.process(%{file_set_id: file_set_2.id}, %{})
+
+      assert {:ok, "s3://test-preservation-checks/pres_check.csv", 2} =
+               PreservationCheckWriter.generate_report(@report_filename)
+    end
   end
 end