Skip to content

Commit a05c6a0

Browse files
willr3johnaohara
willr3
authored and
johnaohara
committed
add multiFilter to run and test labelValues
1 parent 33f0f72 commit a05c6a0

File tree

10 files changed

+460
-99
lines changed

10 files changed

+460
-99
lines changed

‎docs/site/content/en/docs/Tutorials/grafana/index.md

+7-2
Original file line numberDiff line numberDiff line change
@@ -89,15 +89,20 @@ There are 2 ways to filter:
8989
and `{"version":"1.2.3","txRate":2000}` will add the `txRate=2000` requirement.
9090

9191
> curl --query-param "filter={\"version\":\"1.2.3\",\"txRate\":2000}" <horreum>:/api/test/{id}/labelValues
92-
92+
93+
Grafana offers a multi-select option for variables. This sends the options as an array when using `json` encoding.
94+
Horreum will default to looking for a label with an array value instead of any value in the array.
95+
Adding the `multiFilter=true` query parameter allows Horreum to also look for any value in the array and supports Grafana mulit-select.
96+
97+
> curl --query-param "multiFilter=true" --query-param "filter={\"count\":[1,2,4,8]}" <horreum>:/api/test/{id}/labelValues
9398
9499
2. provide a json path (an extractor path from labels) that needs to evaluate to true
95100

96101
For example, if `count` is a label we can pass in `$.count ? (@ > 10 && @ < 20)` to only include datasets where count is between 10 and 20.
97102

98103
> curl --query-param "filter=\"$.count ? (@ > 10 && @ < 20)\"" <horreum>:/api/test/{id}/labelValues
99104
100-
We set the `filter` parameter by editing the Query for the grafana panel but it will depend .
105+
We set the `filter` parameter by editing the Query for the grafana panel.
101106

102107
{{% imgproc json_api_panel_filter Fit "865x331" %}}
103108
Define filter for query

‎docs/site/content/en/openapi/openapi.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -1043,6 +1043,13 @@ paths:
10431043
multiple:
10441044
description: excluding multiple labels
10451045
value: "id,count"
1046+
- name: multiFilter
1047+
in: query
1048+
description: enable filtering for multiple values with an array of values
1049+
schema:
1050+
default: false
1051+
type: boolean
1052+
example: true
10461053
responses:
10471054
"200":
10481055
description: label Values
@@ -2155,6 +2162,13 @@ paths:
21552162
multiple:
21562163
description: excluding multiple labels
21572164
value: "id,count"
2165+
- name: multiFilter
2166+
in: query
2167+
description: enable filtering for multiple values with an array of values
2168+
schema:
2169+
default: false
2170+
type: boolean
2171+
example: true
21582172
responses:
21592173
"200":
21602174
description: OK

‎horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/services/RunService.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,8 @@ Object getData(@PathParam("id") int id,
121121
examples = {
122122
@ExampleObject(name="single", value="id", description = "excluding a single label"),
123123
@ExampleObject(name="multiple", value="id,count", description = "excluding multiple labels")
124-
})
124+
}),
125+
@Parameter(name = "multiFilter", description = "enable filtering for multiple values with an array of values", example = "true")
125126
})
126127
@APIResponses(
127128
value = {
@@ -145,7 +146,8 @@ List<ExportedLabelValues> labelValues(
145146
@QueryParam("limit") @DefaultValue(""+Integer.MAX_VALUE) int limit,
146147
@QueryParam("page") @DefaultValue("0") int page,
147148
@QueryParam("include") @Separator(",") List<String> include,
148-
@QueryParam("exclude") @Separator(",") List<String> exclude);
149+
@QueryParam("exclude") @Separator(",") List<String> exclude,
150+
@QueryParam("multiFilter") @DefaultValue("false") boolean multiFilter);
149151

150152
@GET
151153
@Path("{id}/metadata")

‎horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/services/TestService.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,8 @@ void updateNotifications(@PathParam("id") int id,
227227
examples = {
228228
@ExampleObject(name="single", value="id", description = "excluding a single label"),
229229
@ExampleObject(name="multiple", value="id,count", description = "excluding multiple labels")
230-
})
230+
}),
231+
@Parameter(name = "multiFilter", description = "enable filtering for multiple values with an array of values", example = "true")
231232
})
232233
@APIResponses(
233234
value = { @APIResponse( responseCode = "200",
@@ -247,7 +248,8 @@ List<ExportedLabelValues> labelValues(
247248
@QueryParam("limit") @DefaultValue(""+Integer.MAX_VALUE) int limit,
248249
@QueryParam("page") @DefaultValue("0") int page,
249250
@QueryParam("include") @Separator(",") List<String> include,
250-
@QueryParam("exclude") @Separator(",") List<String> exclude);
251+
@QueryParam("exclude") @Separator(",") List<String> exclude,
252+
@QueryParam("multiFilter") @DefaultValue("false") boolean multiFilter);
251253

252254
@POST
253255
@Consumes(MediaType.APPLICATION_JSON)

‎horreum-backend/src/main/java/io/hyperfoil/tools/horreum/hibernate/JsonbSetType.java

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package io.hyperfoil.tools.horreum.hibernate;
2+
3+
import com.fasterxml.jackson.core.JsonProcessingException;
4+
import com.fasterxml.jackson.databind.ObjectMapper;
5+
import com.fasterxml.jackson.databind.node.ArrayNode;
6+
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
7+
import io.hyperfoil.tools.horreum.svc.Util;
8+
import org.hibernate.HibernateException;
9+
import org.hibernate.engine.spi.SharedSessionContractImplementor;
10+
import org.hibernate.type.CustomType;
11+
import org.hibernate.type.SqlTypes;
12+
import org.hibernate.type.spi.TypeConfiguration;
13+
import org.hibernate.usertype.UserType;
14+
15+
import java.io.Serializable;
16+
import java.sql.*;
17+
import java.util.*;
18+
19+
import static java.lang.String.format;
20+
21+
public class JsonbSetType implements UserType<ArrayNode> {
22+
23+
24+
public static final CustomType INSTANCE = new CustomType<>(new JsonbSetType(), new TypeConfiguration());
25+
26+
@Override
27+
public int getSqlType() {
28+
return SqlTypes.ARRAY;
29+
}
30+
31+
@Override
32+
public Class<ArrayNode> returnedClass() {
33+
return ArrayNode.class;
34+
}
35+
36+
@Override
37+
public boolean equals(ArrayNode x, ArrayNode y) {
38+
return x.equals(y);
39+
}
40+
41+
@Override
42+
public int hashCode(ArrayNode x) {
43+
return Objects.hashCode(x);
44+
}
45+
46+
@Override
47+
public ArrayNode nullSafeGet(ResultSet rs, int position, SharedSessionContractImplementor session, Object owner)
48+
throws SQLException {
49+
if(rs.wasNull())
50+
return null;
51+
Array array = rs.getArray(position);
52+
if (array == null) {
53+
return null;
54+
}
55+
try {
56+
String[] raw = (String[]) array.getArray();
57+
ArrayNode rtrn = JsonNodeFactory.instance.arrayNode();
58+
for(int i=0; i<raw.length; i++){
59+
rtrn.add(Util.toJsonNode(raw[i]));
60+
}
61+
return rtrn;
62+
} catch (final Exception ex) {
63+
throw new RuntimeException("Failed to convert ResultSet to json array: " + ex.getMessage(), ex);
64+
}
65+
}
66+
67+
@Override
68+
public void nullSafeSet(PreparedStatement ps, ArrayNode value, int index, SharedSessionContractImplementor session)
69+
throws SQLException {
70+
if (value == null) {
71+
ps.setNull(index, Types.ARRAY);
72+
return;
73+
}
74+
try {
75+
Set<String> str = new HashSet<>();
76+
value.forEach(v->str.add(v.toString()));
77+
Array array = ps.getConnection().createArrayOf("jsonb", str.toArray());
78+
ps.setObject(index, array);
79+
} catch (final Exception ex) {
80+
throw new RuntimeException(format("Failed to convert JSON to String: %s", ex.getMessage()), ex);
81+
}
82+
}
83+
84+
@Override
85+
public ArrayNode deepCopy(ArrayNode value) throws HibernateException {
86+
if (value == null) {
87+
return null;
88+
}
89+
try {
90+
return (ArrayNode)new ObjectMapper().readTree(value.toString());
91+
} catch (JsonProcessingException e) {
92+
throw new RuntimeException(e);
93+
}
94+
}
95+
96+
@Override
97+
public boolean isMutable() {
98+
return true;
99+
}
100+
101+
@Override
102+
public Serializable disassemble(ArrayNode value) throws HibernateException {
103+
return value.toString();
104+
}
105+
106+
@Override
107+
public ArrayNode assemble(Serializable cached, Object owner) throws HibernateException {
108+
try {
109+
return (ArrayNode)new ObjectMapper().readTree(cached.toString());
110+
} catch (JsonProcessingException e) {
111+
throw new RuntimeException(e);
112+
}
113+
}
114+
115+
public String getName(){ return "jsonb-any";}
116+
}

‎horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java

+26-26
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import io.hyperfoil.tools.horreum.bus.AsyncEventChannels;
2020
import io.hyperfoil.tools.horreum.entity.alerting.DataPointDAO;
2121
import io.hyperfoil.tools.horreum.hibernate.JsonBinaryType;
22+
import io.hyperfoil.tools.horreum.hibernate.JsonbSetType;
2223
import io.hyperfoil.tools.horreum.mapper.DatasetMapper;
2324
import jakarta.annotation.security.PermitAll;
2425
import jakarta.annotation.security.RolesAllowed;
@@ -274,37 +275,27 @@ public Object getData(int id, String token, String schemaUri) {
274275
//this is nearly identical to TestServiceImpl.labelValues (except the return object)
275276
//this reads from the dataset table but provides data specific to the run...
276277
@Override
277-
public List<ExportedLabelValues> labelValues(int runId, String filter, String sort, String direction, int limit, int page, List<String> include, List<String> exclude){
278+
public List<ExportedLabelValues> labelValues(int runId, String filter, String sort, String direction, int limit, int page, List<String> include, List<String> exclude, boolean multiFilter){
278279
List<ExportedLabelValues> rtrn = new ArrayList<>();
279280
Run run = getRun(runId,null);
280281
if(run == null){
281282
throw ServiceException.serverError("Cannot find run "+runId);
282283
}
283284
Object filterObject = Util.getFilterObject(filter);
284-
String filterSql = "";
285-
if(filterObject instanceof JsonNode && ((JsonNode)filterObject).getNodeType() == JsonNodeType.OBJECT){
286-
filterSql = "WHERE "+TestServiceImpl.LABEL_VALUES_FILTER_CONTAINS_JSON;
287-
}else {
288-
Util.CheckResult jsonpathResult = Util.castCheck(filter,"jsonpath",em);
289-
if(jsonpathResult.ok()) {
290-
filterSql = "WHERE "+TestServiceImpl.LABEL_VALUES_FILTER_MATCHES_NOT_NULL;
291-
} else {
292-
if(filter!=null && filter.startsWith("{") && filter.endsWith("}")) {
293-
Util.CheckResult jsonbResult = Util.castCheck(filter, "jsonb", em);
294-
if (!jsonbResult.ok()) {
295-
//we expect this error (because filterObject is not JsonNode
296-
} else {
297-
//this would be a surprise and quite a problem
298-
}
299-
} else {
300-
//how do we report back invalid jsonpath
301-
}
302-
}
285+
286+
TestServiceImpl.FilterDef filterDef = TestServiceImpl.getFilterDef(filter,null,null,multiFilter,(str)->
287+
labelValues(runId,str,sort,direction,limit,page,include,exclude,false),em);
288+
289+
String filterSql = filterDef.sql();
290+
if(filterDef.filterObject()!=null){
291+
filterObject = filterDef.filterObject();
303292
}
293+
304294
if(filterSql.isBlank() && filter != null && !filter.isBlank()){
305295
//TODO there was an error with the filter, do we return that info to the user?
306296
}
307297
String orderSql = "";
298+
308299
String orderDirection = direction.equalsIgnoreCase("ascending") ? "ASC" : "DESC";
309300
if(!sort.isBlank()){
310301
Util.CheckResult jsonpathResult = Util.castCheck(sort, "jsonpath", em);
@@ -315,12 +306,13 @@ public List<ExportedLabelValues> labelValues(int runId, String filter, String so
315306
}
316307
}
317308
String includeExcludeSql = "";
309+
List<String> mutableInclude = new ArrayList<>(include);
310+
318311
if (include!=null && !include.isEmpty()) {
319312
if (exclude != null && !exclude.isEmpty()) {
320-
include = new ArrayList<>(include);
321-
include.removeAll(exclude);
313+
mutableInclude.removeAll(exclude);
322314
}
323-
if (!include.isEmpty()) {
315+
if (!mutableInclude.isEmpty()) {
324316
includeExcludeSql = " AND label.name in :include";
325317
}
326318
}
@@ -349,12 +341,21 @@ SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE la
349341
if(!filterSql.isEmpty()) {
350342
if (filterSql.contains(TestServiceImpl.LABEL_VALUES_FILTER_CONTAINS_JSON)) {
351343
query.setParameter("filter", filterObject, JsonBinaryType.INSTANCE);
352-
} else {
344+
} else if (filterSql.contains(TestServiceImpl.LABEL_VALUES_FILTER_MATCHES_NOT_NULL)){
353345
query.setParameter("filter", filter);
354346
}
355347
}
348+
if(!filterDef.multis().isEmpty() && filterDef.filterObject()!=null){
349+
ObjectNode fullFilterObject = (ObjectNode) Util.getFilterObject(filter);
350+
for(int i=0; i<filterDef.multis().size(); i++){
351+
String key = filterDef.multis().get(i);
352+
ArrayNode value = (ArrayNode) fullFilterObject.get(key);
353+
query.setParameter("key"+i,"$."+key);
354+
query.setParameter("value"+i,value, JsonbSetType.INSTANCE);
355+
}
356+
}
356357
if(includeExcludeSql.contains(":include")){
357-
query.setParameter("include",include);
358+
query.setParameter("include",mutableInclude);
358359
}else if (includeExcludeSql.contains(":exclude")){
359360
query.setParameter("exclude",exclude);
360361
}
@@ -369,7 +370,6 @@ SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE la
369370
.addScalar("datasetId",Integer.class)
370371
.addScalar("start", StandardBasicTypes.INSTANT)
371372
.addScalar("stop", StandardBasicTypes.INSTANT);
372-
373373
//casting because type inference cannot detect there will be two scalars in the result
374374
//TODO replace this with strictly typed entries
375375
((List<Object[]>) query.getResultList()).forEach(objects->{

0 commit comments

Comments
 (0)