Skip to content

Commit 2fc9d35

Browse files
ptitfredptitfred
authored
Validateur NeTEx : validation "à la demande" (#4158)
* Extract helpers for enRoute Chouette Valid * On demand NeTEx validation with enRoute <https://transport.data.gouv.fr/validation?type=netex> * Validation NeTEx : meilleur affichage des erreurs * Affichage des metadata de la validation NeTEx * Validateur NeTEx : meilleur affichage de la durée * NeTEx errors grouped by nature * OnDemand: validateur NeTEx désormais disponible * Meilleur message * Adjust metadata display * Display warning about errors levels * Simplify NeTEx issue template dispatcher * Meilleur message pour les validations rapides * Please the linter * No inline stylesheet for CSP implementation * Fix misusage of heex templates * Fix typo in i18n * i18n: better French version * Formatage de durée : utilisons Cldr.Calendar * Please dialyzer * Please dialyzer * Dialyzer: keep track of the exclusions * Better NeTEx validation introduction * Simplify file generation for on demand validation
1 parent 958de1d commit 2fc9d35

36 files changed

+569
-137
lines changed

‎.dialyzer_ignore.exs

+6-1
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,10 @@
88
# See https://github.com/danielberkompas/cloak_ecto/issues/55
99
{"lib/db/contact.ex", :unknown_type, 0},
1010
{"lib/db/user_feedback.ex", :unknown_type, 0},
11-
{"lib/db/notification.ex", :unknown_type, 0}
11+
{"lib/db/notification.ex", :unknown_type, 0},
12+
13+
# Workaround for "Overloaded contract for Transport.Cldr.Calendar.localize/3
14+
# has overlapping domains; such contracts are currently unsupported and are
15+
# simply ignored."
16+
~r/lib\/cldr.ex/
1217
]

‎apps/shared/lib/cldr.ex

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,5 @@ defmodule Transport.Cldr do
33
Declares a backend for Cldr as required.
44
https://hexdocs.pm/ex_cldr_numbers/readme.html#introduction-and-getting-started
55
"""
6-
use Cldr, locales: ["en", "fr"], providers: [Cldr.Number], default_locale: "fr"
6+
use Cldr, locales: ["en", "fr"], providers: [Cldr.Number, Cldr.Calendar, Cldr.Unit, Cldr.List], default_locale: "fr"
77
end

‎apps/shared/lib/date_time_display.ex

+60
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,66 @@ defmodule Shared.DateTimeDisplay do
104104

105105
def format_datetime_to_paris(nil, _, _), do: ""
106106

107+
@doc """
108+
Formats a duration in seconds to display, according to a locale.
109+
110+
Supported locales: "fr" and "en".
111+
112+
iex> format_duration(1, :en)
113+
"1 second"
114+
iex> format_duration(1, Transport.Cldr.Locale.new!("en"))
115+
"1 second"
116+
iex> format_duration(1, "en")
117+
"1 second"
118+
iex> format_duration(3, "en")
119+
"3 seconds"
120+
iex> format_duration(60, "en")
121+
"1 minute"
122+
iex> format_duration(61, "en")
123+
"1 minute and 1 second"
124+
iex> format_duration(65, "en")
125+
"1 minute and 5 seconds"
126+
iex> format_duration(120, "en")
127+
"2 minutes"
128+
iex> format_duration(125, "en")
129+
"2 minutes and 5 seconds"
130+
iex> format_duration(3601, "en")
131+
"1 hour and 1 second"
132+
iex> format_duration(3661, "en")
133+
"1 hour, 1 minute, and 1 second"
134+
135+
iex> format_duration(1, :fr)
136+
"1 seconde"
137+
iex> format_duration(1, Transport.Cldr.Locale.new!("fr"))
138+
"1 seconde"
139+
iex> format_duration(1, "fr")
140+
"1 seconde"
141+
iex> format_duration(3, "fr")
142+
"3 secondes"
143+
iex> format_duration(60, "fr")
144+
"1 minute"
145+
iex> format_duration(61, "fr")
146+
"1 minute et 1 seconde"
147+
iex> format_duration(65, "fr")
148+
"1 minute et 5 secondes"
149+
iex> format_duration(120, "fr")
150+
"2 minutes"
151+
iex> format_duration(125, "fr")
152+
"2 minutes et 5 secondes"
153+
iex> format_duration(3601, "fr")
154+
"1 heure et 1 seconde"
155+
iex> format_duration(3661, "fr")
156+
"1 heure, 1 minute et 1 seconde"
157+
"""
158+
@spec format_duration(pos_integer(), atom() | Cldr.LanguageTag.t()) :: binary()
159+
def format_duration(duration_in_seconds, locale) do
160+
locale = Cldr.Locale.new!(locale, Transport.Cldr)
161+
162+
duration_in_seconds
163+
|> Cldr.Calendar.Duration.new_from_seconds()
164+
|> Cldr.Calendar.Duration.to_string!(locale: locale)
165+
end
166+
107167
@spec convert_to_paris_time(DateTime.t() | NaiveDateTime.t()) :: DateTime.t()
108168
defp convert_to_paris_time(%DateTime{} = dt) do
109169
case Timex.Timezone.convert(dt, "Europe/Paris") do

‎apps/shared/mix.exs

+3
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ defmodule Shared.MixProject do
5757
# be required no matter what.
5858
{:jason, ">= 0.0.0"},
5959
{:ex_cldr_numbers, "~> 2.0"},
60+
{:ex_cldr_calendars, "~> 1.26"},
61+
{:ex_cldr_lists, "~> 2.11"},
62+
{:ex_cldr_units, "~> 3.17"},
6063
{:cachex, "~> 3.5"},
6164
{:ex_json_schema, "~> 0.10"},
6265
# added because of `TransportWeb.Plugs.AppSignalFilter`

‎apps/transport/client/stylesheets/components/_validation.scss

+29
Original file line numberDiff line numberDiff line change
@@ -123,3 +123,32 @@ details > code {
123123
display: inline;
124124
}
125125
}
126+
127+
table.netex_generic_issue tr.message td {
128+
vertical-align: top;
129+
}
130+
131+
table.netex_generic_issue th:nth-child(1) {
132+
width: 60%;
133+
}
134+
135+
table.netex_generic_issue tr.debug:hover {
136+
background: revert;
137+
}
138+
139+
table.netex_generic_issue tr.debug td {
140+
border-top: none;
141+
padding-top: 0;
142+
}
143+
144+
table.netex_generic_issue tr.debug td summary {
145+
cursor: pointer;
146+
}
147+
148+
table.netex_generic_issue tr.debug td pre {
149+
margin-block: 0;
150+
}
151+
152+
table.netex_generic_issue tr.debug td code {
153+
width: 100%;
154+
}

‎apps/transport/lib/jobs/on_demand_validation_job.ex

+23
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ defmodule Transport.Jobs.OnDemandValidationJob do
1515
alias Transport.DataVisualization
1616
alias Transport.Validators.GTFSRT
1717
alias Transport.Validators.GTFSTransport
18+
alias Transport.Validators.NeTEx
1819
@download_timeout_ms 10_000
1920

2021
@impl Oban.Worker
@@ -71,6 +72,28 @@ defmodule Transport.Jobs.OnDemandValidationJob do
7172
end
7273
end
7374

75+
defp perform_validation(%{"type" => "netex", "permanent_url" => url}) do
76+
validator = NeTEx.validator_name()
77+
78+
case NeTEx.validate(url, []) do
79+
{:error, msg} ->
80+
%{oban_args: %{"state" => "error", "error_reason" => msg}, validator: validator}
81+
82+
{:ok, %{"validations" => validation, "metadata" => metadata}} ->
83+
%{
84+
result: validation,
85+
metadata: metadata,
86+
data_vis: nil,
87+
validator: validator,
88+
validated_data_name: url,
89+
max_error: NeTEx.get_max_severity_error(validation),
90+
oban_args: %{
91+
"state" => "completed"
92+
}
93+
}
94+
end
95+
end
96+
7497
defp perform_validation(%{
7598
"type" => "tableschema",
7699
"permanent_url" => url,

‎apps/transport/lib/transport_web/controllers/resource_controller.ex

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ defmodule TransportWeb.ResourceController do
44
alias Transport.DataVisualization
55
import Ecto.Query
66

7-
import TransportWeb.ResourceView, only: [issue_type: 1, latest_validations_nb_days: 0]
7+
import TransportWeb.ResourceView, only: [latest_validations_nb_days: 0]
88
import TransportWeb.DatasetView, only: [availability_number_days: 0]
99

1010
@enabled_validators MapSet.new([
@@ -144,7 +144,7 @@ defmodule TransportWeb.ResourceController do
144144

145145
issue_type =
146146
case params["issue_type"] do
147-
nil -> issue_type(issues)
147+
nil -> Transport.Validators.GTFSTransport.issue_type(issues)
148148
issue_type -> issue_type
149149
end
150150

‎apps/transport/lib/transport_web/controllers/validation_controller.ex

+33-19
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ defmodule TransportWeb.ValidationController do
22
use TransportWeb, :controller
33
alias DB.{MultiValidation, Repo}
44
alias Transport.DataVisualization
5-
import TransportWeb.ResourceView, only: [issue_type: 1]
65
import Ecto.Query
76

87
def validate(%Plug.Conn{} = conn, %{"upload" => %{"url" => url, "type" => "gbfs"} = params}) do
@@ -98,44 +97,57 @@ defmodule TransportWeb.ValidationController do
9897
unauthorized(conn)
9998

10099
%MultiValidation{oban_args: %{"state" => "completed", "type" => "gtfs"}} = validation ->
101-
current_issues = Transport.Validators.GTFSTransport.get_issues(validation.result, params)
100+
validator = Transport.Validators.GTFSTransport
101+
current_issues = validator.get_issues(validation.result, params)
102102

103103
issue_type =
104104
case params["issue_type"] do
105-
nil -> issue_type(current_issues)
105+
nil -> validator.issue_type(current_issues)
106106
issue_type -> issue_type
107107
end
108108

109109
conn
110-
|> assign(:validation_id, params["id"])
111-
|> assign(:other_resources, [])
112-
|> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params)))
113-
|> assign(
114-
:validation_summary,
115-
Transport.Validators.GTFSTransport.summary(validation.result)
116-
)
117-
|> assign(
118-
:severities_count,
119-
Transport.Validators.GTFSTransport.count_by_severity(validation.result)
120-
)
110+
|> assign_base_validation_details(validator, validation, params, current_issues)
121111
|> assign(:metadata, validation.metadata.metadata)
122112
|> assign(:modes, validation.metadata.modes)
123113
|> assign(:data_vis, data_vis(validation, issue_type))
124-
|> assign(:token, token)
125-
|> render("show.html")
114+
|> render("show_gtfs.html")
115+
116+
%MultiValidation{oban_args: %{"state" => "completed", "type" => "netex"}} = validation ->
117+
validator = Transport.Validators.NeTEx
118+
current_issues = validator.get_issues(validation.result, params)
119+
120+
conn
121+
|> assign_base_validation_details(validator, validation, params, current_issues)
122+
|> assign(:metadata, validation.metadata.metadata)
123+
|> assign(:modes, [])
124+
|> assign(:data_vis, nil)
125+
|> render("show_netex.html")
126126

127127
# Handles waiting for validation to complete, errors and
128128
# validation for schemas
129129
_ ->
130130
live_render(conn, TransportWeb.Live.OnDemandValidationLive,
131131
session: %{
132132
"validation_id" => params["id"],
133+
"issue_type" => params["issue_type"],
133134
"current_url" => validation_path(conn, :show, params["id"], token: token)
134135
}
135136
)
136137
end
137138
end
138139

140+
defp assign_base_validation_details(conn, validator, validation, params, current_issues) do
141+
conn
142+
|> assign(:validator, validator)
143+
|> assign(:validation_id, params["id"])
144+
|> assign(:other_resources, [])
145+
|> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params)))
146+
|> assign(:validation_summary, validator.summary(validation.result))
147+
|> assign(:severities_count, validator.count_by_severity(validation.result))
148+
|> assign(:token, params["token"])
149+
end
150+
139151
defp data_vis(%MultiValidation{} = validation, issue_type) do
140152
data_vis = validation.data_vis[issue_type]
141153
has_features = DataVisualization.has_features(data_vis["geojson"])
@@ -148,9 +160,10 @@ defmodule TransportWeb.ValidationController do
148160
end
149161

150162
defp filepath(type) do
151-
cond do
152-
type == "tableschema" -> Ecto.UUID.generate() <> ".csv"
153-
type in ["jsonschema", "gtfs"] -> Ecto.UUID.generate()
163+
if type == "tableschema" do
164+
Ecto.UUID.generate() <> ".csv"
165+
else
166+
Ecto.UUID.generate()
154167
end
155168
end
156169

@@ -215,6 +228,7 @@ defmodule TransportWeb.ValidationController do
215228
args =
216229
case type do
217230
"gtfs" -> %{"type" => "gtfs"}
231+
"netex" -> %{"type" => "netex"}
218232
schema_name -> %{"schema_name" => schema_name, "type" => schema_type(schema_name)}
219233
end
220234

‎apps/transport/lib/transport_web/live/on_demand_validation_live.ex

+3-3
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ defmodule TransportWeb.Live.OnDemandValidationLive do
3535
schedule_next_update_data()
3636
end
3737

38-
if gtfs_validation_completed?(socket) do
38+
if gtfs_or_netex_validation_completed?(socket) do
3939
redirect(socket, to: socket_value(socket, :current_url))
4040
else
4141
socket
@@ -57,10 +57,10 @@ defmodule TransportWeb.Live.OnDemandValidationLive do
5757
end
5858
end
5959

60-
defp gtfs_validation_completed?(socket) do
60+
defp gtfs_or_netex_validation_completed?(socket) do
6161
case socket_value(socket, :validation) do
6262
%DB.MultiValidation{oban_args: oban_args} ->
63-
oban_args["type"] == "gtfs" and oban_args["state"] == "completed"
63+
oban_args["type"] in ["gtfs", "netex"] and oban_args["state"] == "completed"
6464

6565
_ ->
6666
false

‎apps/transport/lib/transport_web/live/on_demand_validation_select_live.ex

-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@ defmodule TransportWeb.Live.OnDemandValidationSelectLive do
5353

5454
def determine_input_type(type) when type in ["gbfs"], do: "link"
5555
def determine_input_type(type) when type in ["gtfs-rt"], do: "gtfs-rt"
56-
def determine_input_type(type) when type in ["netex"], do: nil
5756
def determine_input_type(_), do: "file"
5857

5958
def handle_event("form_changed", %{"upload" => params, "_target" => target}, socket) do

‎apps/transport/lib/transport_web/live/on_demand_validation_select_live.html.heex

+1-16
Original file line numberDiff line numberDiff line change
@@ -38,28 +38,13 @@
3838
type: "url"
3939
) %>
4040
<% end %>
41-
<%= unless @input_type == "file" or @type == "netex" do %>
41+
<%= unless @input_type == "file" do %>
4242
<%= submit(dgettext("validations", "Validate"), nodiv: true) %>
4343
<% end %>
4444
</.form>
4545
<p :if={@trigger_submit} class="small">
4646
<%= TransportWeb.Gettext.dgettext("validations", "Upload in progress") %>
4747
</p>
48-
<div :if={@type == "netex"} class="container section section-grey" id="netex-explanations">
49-
<p>
50-
<%= raw(
51-
TransportWeb.Gettext.dgettext("validations", "netex-siri-validator", link: "https://greenlight.itxpt.eu")
52-
) %>
53-
</p>
54-
55-
<p>
56-
<%= raw(
57-
TransportWeb.Gettext.dgettext("validations", "doc-eu-formats",
58-
link: "https://doc.transport.data.gouv.fr/documentation/normes-europeennes"
59-
)
60-
) %>
61-
</p>
62-
</div>
6348
</div>
6449
</div>
6550
</section>

‎apps/transport/lib/transport_web/templates/resource/_netex_generic_issue.html.heex

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<table class="table netex_generic_issue">
2+
<tr>
3+
<th><%= dgettext("validations-explanations", "Message") %></th>
4+
<th><%= dgettext("validations-explanations", "Location") %></th>
5+
</tr>
6+
7+
<%= for issue <- @issues do %>
8+
<tr class="message">
9+
<td><%= issue["message"] %></td>
10+
<td>
11+
<%= if is_nil(issue["resource"]) or is_nil(issue["resource"]["filename"]) or is_nil(issue["resource"]["line"]) do %>
12+
<%= dgettext("validations-explanations", "Unknown location") %>
13+
<% else %>
14+
<%= issue["resource"]["filename"] %>:<%= issue["resource"]["line"] %>
15+
<% end %>
16+
</td>
17+
</tr>
18+
<tr class="debug">
19+
<td colspan="2">
20+
<details>
21+
<summary><%= dgettext("validations-explanations", "Details for debugging purposes") %></summary>
22+
<pre><code><%= to_string(Jason.encode!(issue, pretty: true)) %></code></pre>
23+
</details>
24+
</td>
25+
</tr>
26+
<% end %>
27+
</table>

‎apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex

+4-4
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,16 @@ stats = @metadata["stats"] %>
2727
</div>
2828
</li>
2929
<% else %>
30-
<li if={networks != []}>
30+
<li :if={networks != []}>
3131
<div>
3232
<div class="networks-list">
33-
<%= dngettext("validations", "network", "networks", length(@metadata["networks"])) %> :
34-
<strong><%= Enum.join(@metadata["networks"], ", ") %></strong>
33+
<%= dngettext("validations", "network", "networks", length(networks)) %> :
34+
<strong><%= Enum.join(networks, ", ") %></strong>
3535
</div>
3636
</div>
3737
</li>
3838
<% end %>
39-
<li if={length(@modes) > 0}>
39+
<li :if={length(@modes) > 0}>
4040
<%= dngettext("validations", "transport mode", "transport modes", length(@modes)) %> :
4141
<strong><%= Enum.join(@modes, ", ") %></strong>
4242
</li>

0 commit comments

Comments
 (0)