StatsHandler : stocke des statistiques pour les réutilisations (#4483)

Author: AntoineAugusti

apps/transport/lib/db/reuse.ex

@@ -34,6 +34,8 @@ defmodule DB.Reuse do
     many_to_many(:datasets, DB.Dataset, join_through: "reuse_dataset", on_replace: :delete)
   end
 
+  def base_query, do: from(r in __MODULE__, as: :reuse)
+
   def changeset(model, attrs) do
     model
     |> cast(attrs, [

apps/transport/lib/transport/stats_handler.ex

@@ -19,7 +19,7 @@ defmodule Transport.StatsHandler do
   end
 
   defp store_stat_history(key, values, %DateTime{} = timestamp)
-       when key in [:gtfs_rt_types, :climate_resilience_bill_count, :count_geo_data_lines] do
+       when key in [:gtfs_rt_types, :climate_resilience_bill_count, :count_geo_data_lines, :reuses] do
     Enum.map(values, fn {type, count} ->
       store_stat_history("#{key}::#{type}", count, timestamp)
     end)
@@ -88,7 +88,8 @@ defmodule Transport.StatsHandler do
       climate_resilience_bill_count: count_datasets_climate_resilience_bill(),
       nb_siri: count_dataset_with_format("SIRI"),
       nb_siri_lite: count_dataset_with_format("SIRI Lite"),
-      count_geo_data_lines: count_geo_data_lines()
+      count_geo_data_lines: count_geo_data_lines(),
+      reuses: reuses_stats()
     }
     |> Map.merge(gbfs_stats())
   end
@@ -191,6 +192,30 @@ defmodule Transport.StatsHandler do
     |> Enum.into(%{})
   end
 
+  def reuses_stats do
+    reuses_by_type =
+      DB.Reuse.base_query()
+      |> group_by([reuse: r], r.type)
+      |> select([reuse: r], {r.type, count(r.id)})
+      |> DB.Repo.all()
+      |> Enum.into(%{})
+
+    reuses_metrics_sum =
+      DB.Reuse.base_query()
+      |> select([reuse: r], %{
+        sum_metric_discussions: sum(r.metric_discussions),
+        sum_metric_followers: sum(r.metric_followers),
+        sum_metric_views: sum(r.metric_views)
+      })
+      |> DB.Repo.one()
+
+    %{
+      nb_reuses: DB.Repo.aggregate(DB.Reuse.base_query(), :count, :id)
+    }
+    |> Map.merge(reuses_by_type)
+    |> Map.merge(reuses_metrics_sum)
+  end
+
   defp count_datasets_climate_resilience_bill do
     DB.Dataset.base_query()
     |> where([dataset: d], "loi-climat-resilience" in d.custom_tags)

apps/transport/test/support/factory.ex

@@ -41,6 +41,7 @@ defmodule DB.Factory do
       nom: "Grenoble",
       siren: "253800825",
       region: build(:region),
+      population: 1_000,
       # The value must be unique, ExFactory helps us with a named sequence
       composition_res_id: 1000 + sequence("composition_res_id", & &1)
     }
@@ -337,6 +338,33 @@ defmodule DB.Factory do
     }
   end
 
+  def reuse_factory do
+    %DB.Reuse{
+      datagouv_id: sequence(:datagouv_id, &"datagouv_id-#{&1}"),
+      title: sequence(:title, &"Reuse Title #{&1}"),
+      slug: sequence(:slug, &"reuse-slug-#{&1}"),
+      url: sequence(:url, &"http://example.com/reuse/#{&1}"),
+      type: "api",
+      description: "A description of the reuse.",
+      remote_url: sequence(:remote_url, &"http://remote.example.com/reuse/#{&1}"),
+      organization: "Example Organization",
+      organization_id: sequence(:organization_id, &"org-#{&1}"),
+      owner: "Example Owner",
+      owner_id: sequence(:owner_id, &"owner-#{&1}"),
+      image: sequence(:image, &"http://example.com/image/#{&1}.jpg"),
+      featured: false,
+      archived: false,
+      topic: "transport_and_mobility",
+      tags: ["tag1", "tag2"],
+      metric_discussions: 0,
+      metric_datasets: 0,
+      metric_followers: 0,
+      metric_views: 0,
+      created_at: DateTime.utc_now(),
+      last_modified: DateTime.utc_now()
+    }
+  end
+
   def insert_contact(%{} = args \\ %{}) do
     %{
       first_name: "John",

apps/transport/test/transport/stats_handler_test.exs

@@ -47,39 +47,17 @@ defmodule Transport.StatsHandlerTest do
   end
 
   test "climate_resilience_bill_count" do
-    aom = insert(:aom, population: 1_000)
-    insert(:dataset, aom: aom, is_active: false, custom_tags: ["loi-climat-resilience"], type: "public-transit")
-    insert(:dataset, aom: aom, is_active: true, custom_tags: ["loi-climat-resilience"], type: "public-transit")
-    insert(:dataset, aom: aom, is_active: true, custom_tags: ["loi-climat-resilience"], type: "public-transit")
-
-    insert(:dataset,
-      aom: aom,
-      is_active: true,
-      custom_tags: ["loi-climat-resilience", "foo"],
-      type: "low-emission-zones"
-    )
+    insert(:dataset, is_active: false, custom_tags: ["loi-climat-resilience"], type: "public-transit")
+    insert(:dataset, is_active: true, custom_tags: ["loi-climat-resilience"], type: "public-transit")
+    insert(:dataset, is_active: true, custom_tags: ["loi-climat-resilience"], type: "public-transit")
+    insert(:dataset, is_active: true, custom_tags: ["loi-climat-resilience", "foo"], type: "low-emission-zones")
 
     assert %{
              climate_resilience_bill_count: %{
                "low-emission-zones" => 1,
                "public-transit" => 2
              }
            } = compute_stats()
-
-    # Stored as expected in the database
-    store_stats()
-
-    decimal_1 = Decimal.new(1)
-    decimal_2 = Decimal.new(2)
-
-    assert [
-             %DB.StatsHistory{metric: "climate_resilience_bill_count::low-emission-zones", value: ^decimal_1},
-             %DB.StatsHistory{metric: "climate_resilience_bill_count::public-transit", value: ^decimal_2}
-           ] =
-             DB.StatsHistory
-             |> where([s], like(s.metric, "climate_resilience_bill_count%"))
-             |> order_by([s], s.metric)
-             |> DB.Repo.all()
   end
 
   test "store_stats" do
@@ -98,6 +76,16 @@ defmodule Transport.StatsHandlerTest do
       resource_id: insert(:resource, format: "gbfs").id
     )
 
+    # climate_resilience_bill data
+    insert(:dataset, is_active: false, custom_tags: ["loi-climat-resilience"], type: "public-transit")
+    insert(:dataset, is_active: true, custom_tags: ["loi-climat-resilience"], type: "public-transit")
+    insert(:dataset, is_active: true, custom_tags: ["loi-climat-resilience"], type: "public-transit")
+    insert(:dataset, is_active: true, custom_tags: ["loi-climat-resilience", "foo"], type: "low-emission-zones")
+
+    # reuses stats
+    insert(:reuse, type: "api", metric_discussions: 0, metric_followers: 2, metric_views: 5)
+    insert(:reuse, type: "visualization", metric_discussions: 1, metric_followers: 1, metric_views: 10)
+
     stats = compute_stats()
     store_stats()
     assert DB.Repo.aggregate(DB.StatsHistory, :count, :id) >= Enum.count(stats)
@@ -109,18 +97,33 @@ defmodule Transport.StatsHandlerTest do
       |> Map.keys()
       |> Enum.map(&to_string/1)
       |> Enum.reject(
-        &String.starts_with?(&1, ["gtfs_rt_types", "climate_resilience_bill_count", "count_geo_data_lines"])
+        &String.starts_with?(&1, ["gtfs_rt_types", "climate_resilience_bill_count", "count_geo_data_lines", "reuses"])
       )
 
     assert MapSet.subset?(MapSet.new(stats_metrics), MapSet.new(all_metrics))
     assert Enum.member?(all_metrics, "gtfs_rt_types::vehicle_positions")
     assert Enum.member?(all_metrics, "gtfs_rt_types::trip_updates")
     assert Enum.member?(all_metrics, "count_geo_data_lines::irve")
+    assert Enum.member?(all_metrics, "reuses::nb_reuses")
 
-    expected = Decimal.new("2")
+    expected = Decimal.new(2)
     assert %{value: ^expected} = DB.Repo.get_by!(DB.StatsHistory, metric: "gtfs_rt_types::vehicle_positions")
-    expected = Decimal.new("1")
+    expected = Decimal.new(1)
     assert %{value: ^expected} = DB.Repo.get_by!(DB.StatsHistory, metric: "gbfs_v3.0_count")
+    expected = Decimal.new(15)
+    assert %{value: ^expected} = DB.Repo.get_by!(DB.StatsHistory, metric: "reuses::sum_metric_views")
+
+    decimal_1 = Decimal.new(1)
+    decimal_2 = Decimal.new(2)
+
+    assert [
+             %DB.StatsHistory{metric: "climate_resilience_bill_count::low-emission-zones", value: ^decimal_1},
+             %DB.StatsHistory{metric: "climate_resilience_bill_count::public-transit", value: ^decimal_2}
+           ] =
+             DB.StatsHistory
+             |> where([s], like(s.metric, "climate_resilience_bill_count%"))
+             |> order_by([s], s.metric)
+             |> DB.Repo.all()
   end
 
   test "count dataset per format" do
@@ -251,4 +254,20 @@ defmodule Transport.StatsHandlerTest do
              } == gbfs_stats()
     end
   end
+
+  test "reuses_stats" do
+    insert(:reuse, type: "api", metric_discussions: 0, metric_followers: 2, metric_views: 5)
+    insert(:reuse, type: "visualization", metric_discussions: 1, metric_followers: 1, metric_views: 10)
+
+    assert %{
+             :nb_reuses => 2,
+             :sum_metric_discussions => 1,
+             :sum_metric_followers => 3,
+             :sum_metric_views => 15,
+             "api" => 1,
+             "visualization" => 1
+           } = reuses_stats = reuses_stats()
+
+    assert %{reuses: ^reuses_stats} = compute_stats()
+  end
 end