From b9111d04353e39a8b3063832cb9671159fc600e2 Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Tue, 26 Jun 2018 21:42:13 -0700 Subject: [PATCH 01/10] Scripted metric aggregations: add deprecation warning and system property to control legacy params Scripted metric aggregation params._agg/_aggs are replaced by state/states context variables. By default the old params are still present, and a deprecation warning is emitted when Scripted Metric Aggregations are used. A new system property can be used to disable the legacy params. This functionality will be removed in a future revision. --- .../migrate_7_0/aggregations.asciidoc | 3 ++- .../script/ScriptedMetricAggContexts.java | 21 +++++++++++++++++++ .../scripted/InternalScriptedMetric.java | 4 +++- .../ScriptedMetricAggregatorFactory.java | 13 +++++++++--- .../scripted/InternalScriptedMetricTests.java | 2 ++ .../ScriptedMetricAggregatorTests.java | 19 +++++++++++++++++ 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/aggregations.asciidoc b/docs/reference/migration/migrate_7_0/aggregations.asciidoc index b4f29935be9ab..8cc1f81ff4764 100644 --- a/docs/reference/migration/migrate_7_0/aggregations.asciidoc +++ b/docs/reference/migration/migrate_7_0/aggregations.asciidoc @@ -22,4 +22,5 @@ The object used to share aggregation state between the scripts in a Scripted Met Aggregation is now a variable called `state` available in the script context, rather than being provided via the `params` object as `params._agg`. -The old `params._agg` variable is still available as well. +The old `params._agg` variable is still available as well. This can be disabled by setting +the `es.aggregations.enableDeprecatedScriptedMetricAggParam` system property to `false`. diff --git a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java index 774dc95d39977..db08eceb502d9 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java @@ -22,6 +22,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorer; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; @@ -31,6 +33,25 @@ import java.util.Map; public class ScriptedMetricAggContexts { + private static final DeprecationLogger DEPRECATION_LOGGER = + new DeprecationLogger(Loggers.getLogger(ScriptedMetricAggContexts.class)); + + // Public for access from tests + public static final String AGG_PARAM_DEPRECATION_WARNING = + "params._agg/_aggs for scripted metric aggregations are deprecated, use state/states (not in params) instead. " + + "Set system property es.aggregations.enableDeprecatedScriptedMetricAggParam = false to disable this deprecated behavior."; + + public static boolean deprecatedAggParamEnabled() { + boolean enabled = Boolean.parseBoolean( + System.getProperty("es.aggregations.enableDeprecatedScriptedMetricAggParam", "true")); + + if (enabled) { + DEPRECATION_LOGGER.deprecatedAndMaybeLog("enableDeprecatedScriptedMetricAggParams", AGG_PARAM_DEPRECATION_WARNING); + } + + return enabled; + } + private abstract static class ParamsAndStateBase { private final Map params; private final Object state; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java index f4281c063ff2c..4124a8eeb76ad 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java @@ -96,7 +96,9 @@ public InternalAggregation doReduce(List aggregations, Redu } // Add _aggs to params map for backwards compatibility (redundant with a context variable on the ReduceScript created below). - params.put("_aggs", aggregationObjects); + if (ScriptedMetricAggContexts.deprecatedAggParamEnabled()) { + params.put("_aggs", aggregationObjects); + } ScriptedMetricAggContexts.ReduceScript.Factory factory = reduceContext.scriptService().compile( firstAggregation.reduceScript, ScriptedMetricAggContexts.ReduceScript.CONTEXT); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java index 9bd904a07013d..b3509d42b788f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java @@ -83,10 +83,17 @@ public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBu // Add _agg to params map for backwards compatibility (redundant with context variables on the scripts created below). // When this is removed, aggState (as passed to ScriptedMetricAggregator) can be changed to Map, since // it won't be possible to completely replace it with another type as is possible when it's an entry in params. - if (aggParams.containsKey("_agg") == false) { - aggParams.put("_agg", new HashMap()); + Object aggState = new HashMap(); + if (ScriptedMetricAggContexts.deprecatedAggParamEnabled()) { + if (!aggParams.containsKey("_agg")) { + // Add _agg if it wasn't added manually + aggParams.put("_agg", aggState); + } else { + // If it was added manually, also use it for the agg context variable to reduce the likelihood of + // weird behavior due to multiple different variables. + aggState = aggParams.get("_agg"); + } } - Object aggState = aggParams.get("_agg"); final ScriptedMetricAggContexts.InitScript initScript = this.initScript.newInstance( mergeParams(aggParams, initScriptParams), aggState); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java index 584208af4177c..cdb82d4edbacb 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptedMetricAggContexts; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -129,6 +130,7 @@ protected void assertReduced(InternalScriptedMetric reduced, List) reduced.aggregation()).size()); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java index b2a949ceeee1a..835aa0910211d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptedMetricAggContexts; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -169,6 +170,8 @@ public void testNoDocs() throws IOException { assertEquals(0, ((HashMap) scriptedMetric.aggregation()).size()); } } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } /** @@ -195,6 +198,8 @@ public void testScriptedMetricWithoutCombine() throws IOException { assertEquals(numDocs, list.size()); } } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } /** @@ -217,6 +222,8 @@ public void testScriptedMetricWithCombine() throws IOException { assertEquals(numDocs, scriptedMetric.aggregation()); } } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } /** @@ -240,6 +247,8 @@ public void testScriptedMetricWithCombineAccessesScores() throws IOException { assertEquals((double) numDocs, scriptedMetric.aggregation()); } } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testScriptParamsPassedThrough() throws IOException { @@ -259,6 +268,8 @@ public void testScriptParamsPassedThrough() throws IOException { assertEquals(306, scriptedMetric.aggregation()); } } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testConflictingAggAndScriptParams() throws IOException { @@ -282,6 +293,8 @@ public void testConflictingAggAndScriptParams() throws IOException { ex.getMessage()); } } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testSelfReferencingAggStateAfterInit() throws IOException { @@ -299,6 +312,8 @@ public void testSelfReferencingAggStateAfterInit() throws IOException { assertEquals("Iterable object is self-referencing itself (Scripted metric aggs init script)", ex.getMessage()); } } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testSelfReferencingAggStateAfterMap() throws IOException { @@ -319,6 +334,8 @@ public void testSelfReferencingAggStateAfterMap() throws IOException { assertEquals("Iterable object is self-referencing itself (Scripted metric aggs map script)", ex.getMessage()); } } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testSelfReferencingAggStateAfterCombine() throws IOException { @@ -336,6 +353,8 @@ public void testSelfReferencingAggStateAfterCombine() throws IOException { assertEquals("Iterable object is self-referencing itself (Scripted metric aggs combine script)", ex.getMessage()); } } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } /** From bc866824a4c55a676969f287283cd0b58fb5a2eb Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Thu, 12 Jul 2018 20:16:43 -0700 Subject: [PATCH 02/10] Fix minor style issue and docs test failure --- .../aggregations/metrics/scripted-metric-aggregation.asciidoc | 2 ++ .../metrics/scripted/ScriptedMetricAggregatorFactory.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc index c4857699f9805..7ec775a5891dd 100644 --- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc @@ -26,6 +26,7 @@ POST ledger/_search?size=0 -------------------------------------------------- // CONSOLE // TEST[setup:ledger] +// TEST[warning:params._agg/_aggs for scripted metric aggregations are deprecated, use state/states (not in params) instead. Set system property es.aggregations.enableDeprecatedScriptedMetricAggParam = false to disable this deprecated behavior.] <1> `map_script` is the only required parameter @@ -79,6 +80,7 @@ POST ledger/_search?size=0 -------------------------------------------------- // CONSOLE // TEST[setup:ledger,stored_scripted_metric_script] +// TEST[warning:params._agg/_aggs for scripted metric aggregations are deprecated, use state/states (not in params) instead. Set system property es.aggregations.enableDeprecatedScriptedMetricAggParam = false to disable this deprecated behavior.] <1> script parameters for `init`, `map` and `combine` scripts must be specified in a global `params` object so that it can be shared between the scripts. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java index b3509d42b788f..076c29feceae6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java @@ -85,7 +85,7 @@ public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBu // it won't be possible to completely replace it with another type as is possible when it's an entry in params. Object aggState = new HashMap(); if (ScriptedMetricAggContexts.deprecatedAggParamEnabled()) { - if (!aggParams.containsKey("_agg")) { + if (aggParams.containsKey("_agg") == false) { // Add _agg if it wasn't added manually aggParams.put("_agg", aggState); } else { From 7fd0484dcc06a3ff220f2e29b2adb010b2ddad1b Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Tue, 17 Jul 2018 21:04:10 -0700 Subject: [PATCH 03/10] Disable deprecated params._agg/_aggs in tests and revise tests to use state/states instead --- .../elasticsearch/gradle/BuildPlugin.groovy | 2 + .../metrics/ScriptedMetricIT.java | 338 +++++++----------- .../scripted/InternalScriptedMetricTests.java | 6 +- .../ScriptedMetricAggregatorTests.java | 89 ++--- 4 files changed, 168 insertions(+), 267 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 89e10c50ff782..4725b625609ea 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -692,6 +692,8 @@ class BuildPlugin implements Plugin { systemProperty 'tests.security.manager', 'true' systemProperty 'jna.nosys', 'true' systemProperty 'es.scripting.exception_for_missing_value', 'true' + // TODO: remove this deprecation compatibility setting for 7.0 + systemProperty 'es.aggregations.enableDeprecatedScriptedMetricAggParam', 'false' // TODO: remove setting logging level via system property systemProperty 'tests.logger.level', 'WARN' for (Map.Entry property : System.properties.entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java index 13e1489795996..c000b7fb22891 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java @@ -67,6 +67,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; @@ -90,42 +91,57 @@ public static class CustomScriptPlugin extends MockScriptPlugin { protected Map, Object>> pluginScripts() { Map, Object>> scripts = new HashMap<>(); - scripts.put("_agg['count'] = 1", vars -> - aggScript(vars, agg -> ((Map) agg).put("count", 1))); + scripts.put("state['count'] = 1", vars -> + aggScript(vars, state -> state.put("count", 1))); - scripts.put("_agg.add(1)", vars -> - aggScript(vars, agg -> ((List) agg).add(1))); + scripts.put("state.list.add(1)", vars -> + aggScript(vars, state -> { + // Lazily populate state.list for tests without an init script + if (state.containsKey("list") == false) { + state.put("list", new ArrayList()); + } + + ((List) state.get("list")).add(1); + })); - scripts.put("_agg[param1] = param2", vars -> - aggScript(vars, agg -> ((Map) agg).put(XContentMapValues.extractValue("params.param1", vars), + scripts.put("state[param1] = param2", vars -> + aggScript(vars, state -> state.put((String) XContentMapValues.extractValue("params.param1", vars), XContentMapValues.extractValue("params.param2", vars)))); scripts.put("vars.multiplier = 3", vars -> ((Map) vars.get("vars")).put("multiplier", 3)); - scripts.put("_agg.add(vars.multiplier)", vars -> - aggScript(vars, agg -> ((List) agg).add(XContentMapValues.extractValue("vars.multiplier", vars)))); + scripts.put("state.list.add(vars.multiplier)", vars -> + aggScript(vars, state -> { + // Lazily populate state.list for tests without an init script + if (state.containsKey("list") == false) { + state.put("list", new ArrayList()); + } + + ((List) state.get("list")).add(XContentMapValues.extractValue("vars.multiplier", vars)); + })); // Equivalent to: // // newaggregation = []; // sum = 0; // - // for (a in _agg) { - // sum += a + // for (s in state.list) { + // sum += s // }; // // newaggregation.add(sum); // return newaggregation" // - scripts.put("sum agg values as a new aggregation", vars -> { + scripts.put("sum state values as a new aggregation", vars -> { List newAggregation = new ArrayList(); - List agg = (List) vars.get("_agg"); + Map state = (Map) vars.get("state"); + List list = (List) state.get("list"); - if (agg != null) { + if (list != null) { Integer sum = 0; - for (Object a : (List) agg) { - sum += ((Number) a).intValue(); + for (Object s : list) { + sum += ((Number) s).intValue(); } newAggregation.add(sum); } @@ -137,24 +153,41 @@ protected Map, Object>> pluginScripts() { // newaggregation = []; // sum = 0; // - // for (aggregation in _aggs) { - // for (a in aggregation) { - // sum += a + // for (state in states) { + // for (s in state) { + // sum += s // } // }; // // newaggregation.add(sum); // return newaggregation" // - scripts.put("sum aggs of agg values as a new aggregation", vars -> { + scripts.put("sum all states (lists) values as a new aggregation", vars -> { List newAggregation = new ArrayList(); Integer sum = 0; - List aggs = (List) vars.get("_aggs"); - for (Object aggregation : (List) aggs) { - if (aggregation != null) { - for (Object a : (List) aggregation) { - sum += ((Number) a).intValue(); + List> states = (List>) vars.get("states"); + for (List list : states) { + if (list != null) { + for (Object s : list) { + sum += ((Number) s).intValue(); + } + } + } + newAggregation.add(sum); + return newAggregation; + }); + + scripts.put("sum all states' state.list values as a new aggregation", vars -> { + List newAggregation = new ArrayList(); + Integer sum = 0; + + List> states = (List>) vars.get("states"); + for (Map state : states) { + List list = (List) state.get("list"); + if (list != null) { + for (Object s : list) { + sum += ((Number) s).intValue(); } } } @@ -167,25 +200,25 @@ protected Map, Object>> pluginScripts() { // newaggregation = []; // sum = 0; // - // for (aggregation in _aggs) { - // for (a in aggregation) { - // sum += a + // for (state in states) { + // for (s in state) { + // sum += s // } // }; // // newaggregation.add(sum * multiplier); // return newaggregation" // - scripts.put("multiplied sum aggs of agg values as a new aggregation", vars -> { + scripts.put("multiplied sum all states (lists) values as a new aggregation", vars -> { Integer multiplier = (Integer) vars.get("multiplier"); List newAggregation = new ArrayList(); Integer sum = 0; - List aggs = (List) vars.get("_aggs"); - for (Object aggregation : (List) aggs) { - if (aggregation != null) { - for (Object a : (List) aggregation) { - sum += ((Number) a).intValue(); + List> states = (List>) vars.get("states"); + for (List list : states) { + if (list != null) { + for (Object s : list) { + sum += ((Number) s).intValue(); } } } @@ -193,53 +226,12 @@ protected Map, Object>> pluginScripts() { return newAggregation; }); - scripts.put("state.items = new ArrayList()", vars -> - aggContextScript(vars, state -> ((HashMap) state).put("items", new ArrayList()))); - - scripts.put("state.items.add(1)", vars -> - aggContextScript(vars, state -> { - HashMap stateMap = (HashMap) state; - List items = (List) stateMap.get("items"); - items.add(1); - })); - - scripts.put("sum context state values", vars -> { - int sum = 0; - HashMap state = (HashMap) vars.get("state"); - List items = (List) state.get("items"); - - for (Object x : items) { - sum += (Integer)x; - } - - return sum; - }); - - scripts.put("sum context states", vars -> { - Integer sum = 0; - - List states = (List) vars.get("states"); - for (Object state : states) { - sum += ((Number) state).intValue(); - } - - return sum; - }); - return scripts; } - static Object aggScript(Map vars, Consumer fn) { - return aggScript(vars, fn, "_agg"); - } - - static Object aggContextScript(Map vars, Consumer fn) { - return aggScript(vars, fn, "state"); - } - @SuppressWarnings("unchecked") - private static Object aggScript(Map vars, Consumer fn, String stateVarName) { - T aggState = (T) vars.get(stateVarName); + static Map aggScript(Map vars, Consumer> fn) { + Map aggState = (Map) vars.get("state"); fn.accept(aggState); return aggState; } @@ -285,17 +277,17 @@ public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().cluster().preparePutStoredScript() .setId("mapScript_stored") .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + - " \"source\": \"_agg.add(vars.multiplier)\"} }"), XContentType.JSON)); + " \"source\": \"state.list.add(vars.multiplier)\"} }"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() .setId("combineScript_stored") .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + - " \"source\": \"sum agg values as a new aggregation\"} }"), XContentType.JSON)); + " \"source\": \"sum state values as a new aggregation\"} }"), XContentType.JSON)); assertAcked(client().admin().cluster().preparePutStoredScript() .setId("reduceScript_stored") .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + - " \"source\": \"sum aggs of agg values as a new aggregation\"} }"), XContentType.JSON)); + " \"source\": \"sum all states (lists) values as a new aggregation\"} }"), XContentType.JSON)); indexRandom(true, builders); ensureSearchable(); @@ -315,9 +307,10 @@ public void setUp() throws Exception { // the name of the file script is used in test method while the source of the file script // must match a predefined script from CustomScriptPlugin.pluginScripts() method Files.write(scripts.resolve("init_script.mockscript"), "vars.multiplier = 3".getBytes("UTF-8")); - Files.write(scripts.resolve("map_script.mockscript"), "_agg.add(vars.multiplier)".getBytes("UTF-8")); - Files.write(scripts.resolve("combine_script.mockscript"), "sum agg values as a new aggregation".getBytes("UTF-8")); - Files.write(scripts.resolve("reduce_script.mockscript"), "sum aggs of agg values as a new aggregation".getBytes("UTF-8")); + Files.write(scripts.resolve("map_script.mockscript"), "state.list.add(vars.multiplier)".getBytes("UTF-8")); + Files.write(scripts.resolve("combine_script.mockscript"), "sum state values as a new aggregation".getBytes("UTF-8")); + Files.write(scripts.resolve("reduce_script.mockscript"), + "sum all states (lists) values as a new aggregation".getBytes("UTF-8")); } catch (IOException e) { throw new RuntimeException("failed to create scripts"); } @@ -329,7 +322,7 @@ protected Path nodeConfigPath(int nodeOrdinal) { } public void testMap() { - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg['count'] = 1", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state['count'] = 1", Collections.emptyMap()); SearchResponse response = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -365,52 +358,12 @@ public void testMap() { assertThat(numShardsRun, greaterThan(0)); } - public void testExplicitAggParam() { - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", Collections.emptyMap()); - - SearchResponse response = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(scriptedMetric("scripted").params(params).mapScript(mapScript)) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); - long totalCount = 0; - for (Object object : aggregationList) { - assertThat(object, notNullValue()); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - for (Object o : list) { - assertThat(o, notNullValue()); - assertThat(o, instanceOf(Number.class)); - Number numberValue = (Number) o; - assertThat(numberValue, equalTo((Number) 1)); - totalCount += numberValue.longValue(); - } - } - assertThat(totalCount, equalTo(numDocs)); - } - - public void testMapWithParamsAndImplicitAggMap() { + public void testMapWithParams() { // Split the params up between the script and the aggregation. - // Don't put any _agg map in params. Map scriptParams = Collections.singletonMap("param1", "12"); Map aggregationParams = Collections.singletonMap("param2", 1); - // The _agg hashmap will be available even if not declared in the params map - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg[param1] = param2", scriptParams); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state[param1] = param2", scriptParams); SearchResponse response = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -454,7 +407,6 @@ public void testInitMapWithParams() { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); SearchResponse response = client() @@ -466,7 +418,7 @@ public void testInitMapWithParams() { .initScript( new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap())) .mapScript(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, - "_agg.add(vars.multiplier)", Collections.emptyMap()))) + "state.list.add(vars.multiplier)", Collections.emptyMap()))) .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -483,8 +435,11 @@ public void testInitMapWithParams() { long totalCount = 0; for (Object object : aggregationList) { assertThat(object, notNullValue()); - assertThat(object, instanceOf(List.class)); - List list = (List) object; + assertThat(object, instanceOf(HashMap.class)); + Map map = (Map) object; + assertThat(map, hasKey("list")); + assertThat(map.get("list"), instanceOf(List.class)); + List list = (List) map.get("list"); for (Object o : list) { assertThat(o, notNullValue()); assertThat(o, instanceOf(Number.class)); @@ -501,12 +456,11 @@ public void testMapCombineWithParams() { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(1)", Collections.emptyMap()); Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum state values as a new aggregation", Collections.emptyMap()); SearchResponse response = client() .prepareSearch("idx") @@ -553,13 +507,13 @@ public void testInitMapCombineWithParams() { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(vars.multiplier)", + Collections.emptyMap()); Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum state values as a new aggregation", Collections.emptyMap()); SearchResponse response = client() .prepareSearch("idx") @@ -607,15 +561,15 @@ public void testInitMapCombineReduceWithParams() { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(vars.multiplier)", + Collections.emptyMap()); Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum state values as a new aggregation", Collections.emptyMap()); + Script reduceScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, + "sum all states (lists) values as a new aggregation", Collections.emptyMap()); SearchResponse response = client() .prepareSearch("idx") @@ -652,15 +606,15 @@ public void testInitMapCombineReduceGetProperty() throws Exception { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(vars.multiplier)", + Collections.emptyMap()); Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum state values as a new aggregation", Collections.emptyMap()); + Script reduceScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, + "sum all states (lists) values as a new aggregation", Collections.emptyMap()); SearchResponse searchResponse = client() .prepareSearch("idx") @@ -707,14 +661,14 @@ public void testMapCombineReduceWithParams() { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(vars.multiplier)", + Collections.emptyMap()); Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum state values as a new aggregation", Collections.emptyMap()); + Script reduceScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, + "sum all states (lists) values as a new aggregation", Collections.emptyMap()); SearchResponse response = client() .prepareSearch("idx") @@ -749,13 +703,13 @@ public void testInitMapReduceWithParams() { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(vars.multiplier)", + Collections.emptyMap()); + Script reduceScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, + "sum all states' state.list values as a new aggregation", Collections.emptyMap()); SearchResponse response = client() .prepareSearch("idx") @@ -789,12 +743,12 @@ public void testMapReduceWithParams() { Map varsMap = new HashMap<>(); varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(vars.multiplier)", + Collections.emptyMap()); + Script reduceScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, + "sum all states' state.list values as a new aggregation", Collections.emptyMap()); SearchResponse response = client() .prepareSearch("idx") @@ -828,18 +782,18 @@ public void testInitMapCombineReduceWithParamsAndReduceParams() { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); Map reduceParams = new HashMap<>(); reduceParams.put("multiplier", 4); Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(vars.multiplier)", + Collections.emptyMap()); Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "multiplied sum aggs of agg values as a new aggregation", reduceParams); + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum state values as a new aggregation", Collections.emptyMap()); + Script reduceScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, + "multiplied sum all states (lists) values as a new aggregation", reduceParams); SearchResponse response = client() .prepareSearch("idx") @@ -875,7 +829,6 @@ public void testInitMapCombineReduceWithParamsStored() { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); SearchResponse response = client() @@ -916,15 +869,15 @@ public void testInitMapCombineReduceWithParamsAsSubAgg() { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(vars.multiplier)", + Collections.emptyMap()); Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum state values as a new aggregation", Collections.emptyMap()); + Script reduceScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, + "sum all states (lists) values as a new aggregation", Collections.emptyMap()); SearchResponse response = client() .prepareSearch("idx") @@ -977,15 +930,15 @@ public void testEmptyAggregation() throws Exception { varsMap.put("multiplier", 1); Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(vars.multiplier)", + Collections.emptyMap()); Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum state values as a new aggregation", Collections.emptyMap()); + Script reduceScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, + "sum all states (lists) values as a new aggregation", Collections.emptyMap()); SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) @@ -1021,7 +974,7 @@ public void testEmptyAggregation() throws Exception { * not using a script does get cached. */ public void testDontCacheScripts() throws Exception { - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg['count'] = 1", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state['count'] = 1", Collections.emptyMap()); assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) .get()); @@ -1047,7 +1000,7 @@ public void testDontCacheScripts() throws Exception { public void testConflictingAggAndScriptParams() { Map params = Collections.singletonMap("param1", "12"); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", params); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.list.add(1)", params); SearchRequestBuilder builder = client().prepareSearch("idx") .setQuery(matchAllQuery()) @@ -1056,37 +1009,4 @@ public void testConflictingAggAndScriptParams() { SearchPhaseExecutionException ex = expectThrows(SearchPhaseExecutionException.class, builder::get); assertThat(ex.getCause().getMessage(), containsString("Parameter name \"param1\" used in both aggregation and script parameters")); } - - public void testAggFromContext() { - Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.items = new ArrayList()", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "state.items.add(1)", Collections.emptyMap()); - Script combineScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum context state values", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum context states", - Collections.emptyMap()); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .initScript(initScript) - .mapScript(mapScript) - .combineScript(combineScript) - .reduceScript(reduceScript)) - .get(); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(Integer.class)); - Integer aggResult = (Integer) scriptedMetricAggregation.aggregation(); - long totalAgg = aggResult.longValue(); - assertThat(totalAgg, equalTo(numDocs)); - } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java index cdb82d4edbacb..70ddacf5698b2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptedMetricAggContexts; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -108,7 +107,7 @@ private static Object randomValue(Supplier[] valueTypes, int level) { /** * Mock of the script service. The script that is run looks at the - * "_aggs" parameter visible when executing the script and simply returns the count. + * "states" context variable visible when executing the script and simply returns the count. * This should be equal to the number of input InternalScriptedMetrics that are reduced * in total. */ @@ -117,7 +116,7 @@ protected ScriptService mockScriptService() { // mock script always retuns the size of the input aggs list as result @SuppressWarnings("unchecked") MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, - Collections.singletonMap(REDUCE_SCRIPT_NAME, script -> ((List) script.get("_aggs")).size())); + Collections.singletonMap(REDUCE_SCRIPT_NAME, script -> ((List) script.get("states")).size())); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); } @@ -130,7 +129,6 @@ protected void assertReduced(InternalScriptedMetric reduced, List) reduced.aggregation()).size()); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java index 835aa0910211d..a9ef4a91ea32a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptedMetricAggContexts; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -86,72 +85,72 @@ public class ScriptedMetricAggregatorTests extends AggregatorTestCase { @SuppressWarnings("unchecked") public static void initMockScripts() { SCRIPTS.put("initScript", params -> { - Map agg = (Map) params.get("_agg"); - agg.put("collector", new ArrayList()); - return agg; + Map state = (Map) params.get("state"); + state.put("collector", new ArrayList()); + return state; }); SCRIPTS.put("mapScript", params -> { - Map agg = (Map) params.get("_agg"); - ((List) agg.get("collector")).add(1); // just add 1 for each doc the script is run on - return agg; + Map state = (Map) params.get("state"); + ((List) state.get("collector")).add(1); // just add 1 for each doc the script is run on + return state; }); SCRIPTS.put("combineScript", params -> { - Map agg = (Map) params.get("_agg"); - return ((List) agg.get("collector")).stream().mapToInt(Integer::intValue).sum(); + Map state = (Map) params.get("state"); + return ((List) state.get("collector")).stream().mapToInt(Integer::intValue).sum(); }); SCRIPTS.put("initScriptScore", params -> { - Map agg = (Map) params.get("_agg"); - agg.put("collector", new ArrayList()); - return agg; + Map state = (Map) params.get("state"); + state.put("collector", new ArrayList()); + return state; }); SCRIPTS.put("mapScriptScore", params -> { - Map agg = (Map) params.get("_agg"); - ((List) agg.get("collector")).add(((Number) params.get("_score")).doubleValue()); - return agg; + Map state = (Map) params.get("state"); + ((List) state.get("collector")).add(((Number) params.get("_score")).doubleValue()); + return state; }); SCRIPTS.put("combineScriptScore", params -> { - Map agg = (Map) params.get("_agg"); - return ((List) agg.get("collector")).stream().mapToDouble(Double::doubleValue).sum(); + Map state = (Map) params.get("state"); + return ((List) state.get("collector")).stream().mapToDouble(Double::doubleValue).sum(); }); SCRIPTS.put("initScriptParams", params -> { - Map agg = (Map) params.get("_agg"); + Map state = (Map) params.get("state"); Integer initialValue = (Integer)params.get("initialValue"); ArrayList collector = new ArrayList<>(); collector.add(initialValue); - agg.put("collector", collector); - return agg; + state.put("collector", collector); + return state; }); SCRIPTS.put("mapScriptParams", params -> { - Map agg = (Map) params.get("_agg"); + Map state = (Map) params.get("state"); Integer itemValue = (Integer) params.get("itemValue"); - ((List) agg.get("collector")).add(itemValue); - return agg; + ((List) state.get("collector")).add(itemValue); + return state; }); SCRIPTS.put("combineScriptParams", params -> { - Map agg = (Map) params.get("_agg"); + Map state = (Map) params.get("state"); int divisor = ((Integer) params.get("divisor")); - return ((List) agg.get("collector")).stream().mapToInt(Integer::intValue).map(i -> i / divisor).sum(); + return ((List) state.get("collector")).stream().mapToInt(Integer::intValue).map(i -> i / divisor).sum(); }); SCRIPTS.put("initScriptSelfRef", params -> { - Map agg = (Map) params.get("_agg"); - agg.put("collector", new ArrayList()); - agg.put("selfRef", agg); - return agg; + Map state = (Map) params.get("state"); + state.put("collector", new ArrayList()); + state.put("selfRef", state); + return state; }); SCRIPTS.put("mapScriptSelfRef", params -> { - Map agg = (Map) params.get("_agg"); - agg.put("selfRef", agg); - return agg; + Map state = (Map) params.get("state"); + state.put("selfRef", state); + return state; }); SCRIPTS.put("combineScriptSelfRef", params -> { - Map agg = (Map) params.get("_agg"); - agg.put("selfRef", agg); - return agg; + Map state = (Map) params.get("state"); + state.put("selfRef", state); + return state; }); } @@ -170,12 +169,10 @@ public void testNoDocs() throws IOException { assertEquals(0, ((HashMap) scriptedMetric.aggregation()).size()); } } - - assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } /** - * without combine script, the "_aggs" map should contain a list of the size of the number of documents matched + * without combine script, the "states" map should contain a list of the size of the number of documents matched */ public void testScriptedMetricWithoutCombine() throws IOException { try (Directory directory = newDirectory()) { @@ -198,8 +195,6 @@ public void testScriptedMetricWithoutCombine() throws IOException { assertEquals(numDocs, list.size()); } } - - assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } /** @@ -222,8 +217,6 @@ public void testScriptedMetricWithCombine() throws IOException { assertEquals(numDocs, scriptedMetric.aggregation()); } } - - assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } /** @@ -247,8 +240,6 @@ public void testScriptedMetricWithCombineAccessesScores() throws IOException { assertEquals((double) numDocs, scriptedMetric.aggregation()); } } - - assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testScriptParamsPassedThrough() throws IOException { @@ -268,8 +259,6 @@ public void testScriptParamsPassedThrough() throws IOException { assertEquals(306, scriptedMetric.aggregation()); } } - - assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testConflictingAggAndScriptParams() throws IOException { @@ -293,8 +282,6 @@ public void testConflictingAggAndScriptParams() throws IOException { ex.getMessage()); } } - - assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testSelfReferencingAggStateAfterInit() throws IOException { @@ -312,8 +299,6 @@ public void testSelfReferencingAggStateAfterInit() throws IOException { assertEquals("Iterable object is self-referencing itself (Scripted metric aggs init script)", ex.getMessage()); } } - - assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testSelfReferencingAggStateAfterMap() throws IOException { @@ -334,8 +319,6 @@ public void testSelfReferencingAggStateAfterMap() throws IOException { assertEquals("Iterable object is self-referencing itself (Scripted metric aggs map script)", ex.getMessage()); } } - - assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } public void testSelfReferencingAggStateAfterCombine() throws IOException { @@ -353,8 +336,6 @@ public void testSelfReferencingAggStateAfterCombine() throws IOException { assertEquals("Iterable object is self-referencing itself (Scripted metric aggs combine script)", ex.getMessage()); } } - - assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); } /** From 8185ac304607022fcf770fc86a2a7acb1a6dc2f5 Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Tue, 17 Jul 2018 22:19:09 -0700 Subject: [PATCH 04/10] Add integration test covering deprecated scripted metrics aggs params._agg/_aggs access --- server/build.gradle | 15 + .../ScriptedMetricAggParamsV6CompatIT.java | 1022 +++++++++++++++++ 2 files changed, 1037 insertions(+) create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggParamsV6CompatIT.java diff --git a/server/build.gradle b/server/build.gradle index da60bca5a3e81..6e37444734a9a 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -334,7 +334,22 @@ if (isEclipse == false || project.path == ":server-tests") { classpath = project.test.classpath testClassesDirs = project.test.testClassesDirs include '**/*IT.class' + // TODO: remove ScriptedMetricAggParamsV6CompatIT for 7.0 + exclude '**/ScriptedMetricAggParamsV6CompatIT.class' } + + // TODO: remove this task dedicated to ScriptedMetricAggParamsV6CompatIT for 7.0 + task scriptedMetricAggDeprecatedParamIntegTest(type: RandomizedTestingTask, + description: 'Dedicated backwards compatibility integration test', + dependsOn: integTest.dependsOn.collect()) { + configure(BuildPlugin.commonTestConfig(project)) + classpath = integTest.classpath + testClassesDirs = integTest.testClassesDirs + systemProperty 'es.aggregations.enableDeprecatedScriptedMetricAggParam', 'true' + include '**/ScriptedMetricAggParamsV6CompatIT.class' + } + integTest.dependsOn scriptedMetricAggDeprecatedParamIntegTest + check.dependsOn integTest integTest.mustRunAfter test } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggParamsV6CompatIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggParamsV6CompatIT.java new file mode 100644 index 0000000000000..73ad84036ae0c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggParamsV6CompatIT.java @@ -0,0 +1,1022 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.bucket.global.Global; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; +import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetric; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.scriptedMetric; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; + +// This is a clone of relevant tests from ScriptedMetricIT from before they were converted away from the deprecated +// params._agg/_aggs state storage to the 'state' context variable. This test runs with the deprecated behavior enabled +// to ensure that it doesn't break. +// TODO: Remove this class for 7.0 when the deprecated behavior is removed. +@ClusterScope(scope = Scope.SUITE) +@ESIntegTestCase.SuiteScopeTestCase +public class ScriptedMetricAggParamsV6CompatIT extends ESIntegTestCase { + + private static long numDocs; + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = new HashMap<>(); + + scripts.put("_agg['count'] = 1", vars -> + aggScript(vars, agg -> ((Map) agg).put("count", 1))); + + scripts.put("_agg.add(1)", vars -> + aggScript(vars, agg -> ((List) agg).add(1))); + + scripts.put("_agg[param1] = param2", vars -> + aggScript(vars, agg -> ((Map) agg).put(XContentMapValues.extractValue("params.param1", vars), + XContentMapValues.extractValue("params.param2", vars)))); + + scripts.put("vars.multiplier = 3", vars -> + ((Map) vars.get("vars")).put("multiplier", 3)); + + scripts.put("_agg.add(vars.multiplier)", vars -> + aggScript(vars, agg -> ((List) agg).add(XContentMapValues.extractValue("vars.multiplier", vars)))); + + // Equivalent to: + // + // newaggregation = []; + // sum = 0; + // + // for (a in _agg) { + // sum += a + // }; + // + // newaggregation.add(sum); + // return newaggregation" + // + scripts.put("sum agg values as a new aggregation", vars -> { + List newAggregation = new ArrayList(); + List agg = (List) vars.get("_agg"); + + if (agg != null) { + Integer sum = 0; + for (Object a : (List) agg) { + sum += ((Number) a).intValue(); + } + newAggregation.add(sum); + } + return newAggregation; + }); + + // Equivalent to: + // + // newaggregation = []; + // sum = 0; + // + // for (aggregation in _aggs) { + // for (a in aggregation) { + // sum += a + // } + // }; + // + // newaggregation.add(sum); + // return newaggregation" + // + scripts.put("sum aggs of agg values as a new aggregation", vars -> { + List newAggregation = new ArrayList(); + Integer sum = 0; + + List aggs = (List) vars.get("_aggs"); + for (Object aggregation : (List) aggs) { + if (aggregation != null) { + for (Object a : (List) aggregation) { + sum += ((Number) a).intValue(); + } + } + } + newAggregation.add(sum); + return newAggregation; + }); + + // Equivalent to: + // + // newaggregation = []; + // sum = 0; + // + // for (aggregation in _aggs) { + // for (a in aggregation) { + // sum += a + // } + // }; + // + // newaggregation.add(sum * multiplier); + // return newaggregation" + // + scripts.put("multiplied sum aggs of agg values as a new aggregation", vars -> { + Integer multiplier = (Integer) vars.get("multiplier"); + List newAggregation = new ArrayList(); + Integer sum = 0; + + List aggs = (List) vars.get("_aggs"); + for (Object aggregation : (List) aggs) { + if (aggregation != null) { + for (Object a : (List) aggregation) { + sum += ((Number) a).intValue(); + } + } + } + newAggregation.add(sum * multiplier); + return newAggregation; + }); + + return scripts; + } + + @SuppressWarnings("unchecked") + private static Object aggScript(Map vars, Consumer fn) { + T aggState = (T) vars.get("_agg"); + fn.accept(aggState); + return aggState; + } + } + + @Override + public void setupSuiteScopeCluster() throws Exception { + createIndex("idx"); + + List builders = new ArrayList<>(); + + numDocs = randomIntBetween(10, 100); + for (int i = 0; i < numDocs; i++) { + builders.add(client().prepareIndex("idx", "type", "" + i).setSource( + jsonBuilder().startObject().field("value", randomAlphaOfLengthBetween(5, 15)) + .field("l_value", i).endObject())); + } + indexRandom(true, builders); + + // creating an index to test the empty buckets functionality. The way it + // works is by indexing + // two docs {value: 0} and {value : 2}, then building a histogram agg + // with interval 1 and with empty + // buckets computed.. the empty bucket is the one associated with key + // "1". then each test will have + // to check that this bucket exists with the appropriate sub + // aggregations. + prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer").execute().actionGet(); + builders = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + i).setSource( + jsonBuilder().startObject().field("value", i * 2).endObject())); + } + + // When using the MockScriptPlugin we can map Stored scripts to inline scripts: + // the id of the stored script is used in test method while the source of the stored script + // must match a predefined script from CustomScriptPlugin.pluginScripts() method + assertAcked(client().admin().cluster().preparePutStoredScript() + .setId("initScript_stored") + .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + + " \"source\": \"vars.multiplier = 3\"} }"), XContentType.JSON)); + + assertAcked(client().admin().cluster().preparePutStoredScript() + .setId("mapScript_stored") + .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + + " \"source\": \"_agg.add(vars.multiplier)\"} }"), XContentType.JSON)); + + assertAcked(client().admin().cluster().preparePutStoredScript() + .setId("combineScript_stored") + .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + + " \"source\": \"sum agg values as a new aggregation\"} }"), XContentType.JSON)); + + assertAcked(client().admin().cluster().preparePutStoredScript() + .setId("reduceScript_stored") + .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + + " \"source\": \"sum aggs of agg values as a new aggregation\"} }"), XContentType.JSON)); + + indexRandom(true, builders); + ensureSearchable(); + } + + private Path config; + + @Before + public void setUp() throws Exception { + super.setUp(); + config = createTempDir().resolve("config"); + final Path scripts = config.resolve("scripts"); + + try { + Files.createDirectories(scripts); + // When using the MockScriptPlugin we can map File scripts to inline scripts: + // the name of the file script is used in test method while the source of the file script + // must match a predefined script from CustomScriptPlugin.pluginScripts() method + Files.write(scripts.resolve("init_script.mockscript"), "vars.multiplier = 3".getBytes("UTF-8")); + Files.write(scripts.resolve("map_script.mockscript"), "_agg.add(vars.multiplier)".getBytes("UTF-8")); + Files.write(scripts.resolve("combine_script.mockscript"), "sum agg values as a new aggregation".getBytes("UTF-8")); + Files.write(scripts.resolve("reduce_script.mockscript"), "sum aggs of agg values as a new aggregation".getBytes("UTF-8")); + } catch (IOException e) { + throw new RuntimeException("failed to create scripts"); + } + } + + @Override + protected Path nodeConfigPath(int nodeOrdinal) { + return config; + } + + public void testMap() { + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg['count'] = 1", Collections.emptyMap()); + + SearchResponse response = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(scriptedMetric("scripted").mapScript(mapScript)) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); + int numShardsRun = 0; + for (Object object : aggregationList) { + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Map.class)); + Map map = (Map) object; + assertThat(map.size(), lessThanOrEqualTo(1)); + if (map.size() == 1) { + assertThat(map.get("count"), notNullValue()); + assertThat(map.get("count"), instanceOf(Number.class)); + assertThat(map.get("count"), equalTo((Number) 1)); + numShardsRun++; + } + } + // We don't know how many shards will have documents but we need to make + // sure that at least one shard ran the map script + assertThat(numShardsRun, greaterThan(0)); + } + + public void testExplicitAggParam() { + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", Collections.emptyMap()); + + SearchResponse response = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(scriptedMetric("scripted").params(params).mapScript(mapScript)) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); + long totalCount = 0; + for (Object object : aggregationList) { + assertThat(object, notNullValue()); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + for (Object o : list) { + assertThat(o, notNullValue()); + assertThat(o, instanceOf(Number.class)); + Number numberValue = (Number) o; + assertThat(numberValue, equalTo((Number) 1)); + totalCount += numberValue.longValue(); + } + } + assertThat(totalCount, equalTo(numDocs)); + } + + public void testMapWithParamsAndImplicitAggMap() { + // Split the params up between the script and the aggregation. + // Don't put any _agg map in params. + Map scriptParams = Collections.singletonMap("param1", "12"); + Map aggregationParams = Collections.singletonMap("param2", 1); + + // The _agg hashmap will be available even if not declared in the params map + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg[param1] = param2", scriptParams); + + SearchResponse response = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(scriptedMetric("scripted").params(aggregationParams).mapScript(mapScript)) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); + int numShardsRun = 0; + for (Object object : aggregationList) { + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Map.class)); + Map map = (Map) object; + for (Map.Entry entry : map.entrySet()) { + assertThat(entry, notNullValue()); + assertThat(entry.getKey(), notNullValue()); + assertThat(entry.getKey(), instanceOf(String.class)); + assertThat(entry.getValue(), notNullValue()); + assertThat(entry.getValue(), instanceOf(Number.class)); + String stringValue = (String) entry.getKey(); + assertThat(stringValue, equalTo("12")); + Number numberValue = (Number) entry.getValue(); + assertThat(numberValue, equalTo((Number) 1)); + numShardsRun++; + } + } + assertThat(numShardsRun, greaterThan(0)); + } + + public void testInitMapWithParams() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + scriptedMetric("scripted") + .params(params) + .initScript( + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap())) + .mapScript(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, + "_agg.add(vars.multiplier)", Collections.emptyMap()))) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); + long totalCount = 0; + for (Object object : aggregationList) { + assertThat(object, notNullValue()); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + for (Object o : list) { + assertThat(o, notNullValue()); + assertThat(o, instanceOf(Number.class)); + Number numberValue = (Number) o; + assertThat(numberValue, equalTo((Number) 3)); + totalCount += numberValue.longValue(); + } + } + assertThat(totalCount, equalTo(numDocs * 3)); + } + + public void testMapCombineWithParams() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", Collections.emptyMap()); + Script combineScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + scriptedMetric("scripted") + .params(params) + .mapScript(mapScript) + .combineScript(combineScript)) + .execute().actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); + long totalCount = 0; + for (Object object : aggregationList) { + assertThat(object, notNullValue()); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + for (Object o : list) { + assertThat(o, notNullValue()); + assertThat(o, instanceOf(Number.class)); + Number numberValue = (Number) o; + // A particular shard may not have any documents stored on it so + // we have to assume the lower bound may be 0. The check at the + // bottom of the test method will make sure the count is correct + assertThat(numberValue.longValue(), allOf(greaterThanOrEqualTo(0L), lessThanOrEqualTo(numDocs))); + totalCount += numberValue.longValue(); + } + } + assertThat(totalCount, equalTo(numDocs)); + } + + public void testInitMapCombineWithParams() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script combineScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + scriptedMetric("scripted") + .params(params) + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript)) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); + long totalCount = 0; + for (Object object : aggregationList) { + assertThat(object, notNullValue()); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + for (Object o : list) { + assertThat(o, notNullValue()); + assertThat(o, instanceOf(Number.class)); + Number numberValue = (Number) o; + // A particular shard may not have any documents stored on it so + // we have to assume the lower bound may be 0. The check at the + // bottom of the test method will make sure the count is correct + assertThat(numberValue.longValue(), allOf(greaterThanOrEqualTo(0L), lessThanOrEqualTo(numDocs * 3))); + totalCount += numberValue.longValue(); + } + } + assertThat(totalCount, equalTo(numDocs * 3)); + } + + public void testInitMapCombineReduceWithParams() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script combineScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + Script reduceScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + scriptedMetric("scripted") + .params(params) + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript)) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(1)); + Object object = aggregationList.get(0); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Number.class)); + assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); + } + + @SuppressWarnings("rawtypes") + public void testInitMapCombineReduceGetProperty() throws Exception { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script combineScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + Script reduceScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + + SearchResponse searchResponse = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + global("global") + .subAggregation( + scriptedMetric("scripted") + .params(params) + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript))) + .get(); + + assertSearchResponse(searchResponse); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocs)); + + Global global = searchResponse.getAggregations().get("global"); + assertThat(global, notNullValue()); + assertThat(global.getName(), equalTo("global")); + assertThat(global.getDocCount(), equalTo(numDocs)); + assertThat(global.getAggregations(), notNullValue()); + assertThat(global.getAggregations().asMap().size(), equalTo(1)); + + ScriptedMetric scriptedMetricAggregation = global.getAggregations().get("scripted"); + assertThat(scriptedMetricAggregation, notNullValue()); + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(1)); + Object object = aggregationList.get(0); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Number.class)); + assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); + assertThat(((InternalAggregation)global).getProperty("scripted"), sameInstance(scriptedMetricAggregation)); + assertThat((List) ((InternalAggregation)global).getProperty("scripted.value"), sameInstance((List) aggregationList)); + assertThat((List) ((InternalAggregation)scriptedMetricAggregation).getProperty("value"), sameInstance((List) aggregationList)); + } + + public void testMapCombineReduceWithParams() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script combineScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + Script reduceScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + scriptedMetric("scripted") + .params(params) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript)) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(1)); + Object object = aggregationList.get(0); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Number.class)); + assertThat(((Number) object).longValue(), equalTo(numDocs)); + } + + public void testInitMapReduceWithParams() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script reduceScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + scriptedMetric("scripted") + .params(params) + .initScript(initScript) + .mapScript(mapScript) + .reduceScript(reduceScript)) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(1)); + Object object = aggregationList.get(0); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Number.class)); + assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); + } + + public void testMapReduceWithParams() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script reduceScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + scriptedMetric("scripted") + .params(params) + .mapScript(mapScript) + .reduceScript(reduceScript)) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(1)); + Object object = aggregationList.get(0); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Number.class)); + assertThat(((Number) object).longValue(), equalTo(numDocs)); + } + + public void testInitMapCombineReduceWithParamsAndReduceParams() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Map reduceParams = new HashMap<>(); + reduceParams.put("multiplier", 4); + + Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script combineScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + Script reduceScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "multiplied sum aggs of agg values as a new aggregation", reduceParams); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + scriptedMetric("scripted") + .params(params) + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript)) + .execute().actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(1)); + Object object = aggregationList.get(0); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Number.class)); + assertThat(((Number) object).longValue(), equalTo(numDocs * 12)); + } + + public void testInitMapCombineReduceWithParamsStored() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation( + scriptedMetric("scripted") + .params(params) + .initScript( + new Script(ScriptType.STORED, null, "initScript_stored", Collections.emptyMap())) + .mapScript( + new Script(ScriptType.STORED, null, "mapScript_stored", Collections.emptyMap())) + .combineScript( + new Script(ScriptType.STORED, null, "combineScript_stored", Collections.emptyMap())) + .reduceScript( + new Script(ScriptType.STORED, null, "reduceScript_stored", Collections.emptyMap()))) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + + Aggregation aggregation = response.getAggregations().get("scripted"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(1)); + Object object = aggregationList.get(0); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Number.class)); + assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); + } + + public void testInitMapCombineReduceWithParamsAsSubAgg() { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script combineScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + Script reduceScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + + SearchResponse response = client() + .prepareSearch("idx") + .setQuery(matchAllQuery()).setSize(1000) + .addAggregation( + histogram("histo") + .field("l_value") + .interval(1) + .subAggregation( + scriptedMetric("scripted") + .params(params) + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript))) + .get(); + assertSearchResponse(response); + assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); + Aggregation aggregation = response.getAggregations().get("histo"); + assertThat(aggregation, notNullValue()); + assertThat(aggregation, instanceOf(Histogram.class)); + Histogram histoAgg = (Histogram) aggregation; + assertThat(histoAgg.getName(), equalTo("histo")); + List buckets = histoAgg.getBuckets(); + assertThat(buckets, notNullValue()); + for (Bucket b : buckets) { + assertThat(b, notNullValue()); + assertThat(b.getDocCount(), equalTo(1L)); + Aggregations subAggs = b.getAggregations(); + assertThat(subAggs, notNullValue()); + assertThat(subAggs.asList().size(), equalTo(1)); + Aggregation subAgg = subAggs.get("scripted"); + assertThat(subAgg, notNullValue()); + assertThat(subAgg, instanceOf(ScriptedMetric.class)); + ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) subAgg; + assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); + assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); + assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); + List aggregationList = (List) scriptedMetricAggregation.aggregation(); + assertThat(aggregationList.size(), equalTo(1)); + Object object = aggregationList.get(0); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Number.class)); + assertThat(((Number) object).longValue(), equalTo(3L)); + } + } + + public void testEmptyAggregation() throws Exception { + Map varsMap = new HashMap<>(); + varsMap.put("multiplier", 1); + + Map params = new HashMap<>(); + params.put("_agg", new ArrayList<>()); + params.put("vars", varsMap); + + Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); + Script combineScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); + Script reduceScript = + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); + + SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") + .setQuery(matchAllQuery()) + .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0) + .subAggregation( + scriptedMetric("scripted") + .params(params) + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript))) + .get(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L)); + Histogram histo = searchResponse.getAggregations().get("histo"); + assertThat(histo, notNullValue()); + Histogram.Bucket bucket = histo.getBuckets().get(1); + assertThat(bucket, notNullValue()); + + ScriptedMetric scriptedMetric = bucket.getAggregations().get("scripted"); + assertThat(scriptedMetric, notNullValue()); + assertThat(scriptedMetric.getName(), equalTo("scripted")); + assertThat(scriptedMetric.aggregation(), notNullValue()); + assertThat(scriptedMetric.aggregation(), instanceOf(List.class)); + @SuppressWarnings("unchecked") // We'll just get a ClassCastException a couple lines down if we're wrong, its ok. + List aggregationResult = (List) scriptedMetric.aggregation(); + assertThat(aggregationResult.size(), equalTo(1)); + assertThat(aggregationResult.get(0), equalTo(0)); + } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg['count'] = 1", Collections.emptyMap()); + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(scriptedMetric("foo").mapScript(mapScript)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + } + + public void testConflictingAggAndScriptParams() { + Map params = Collections.singletonMap("param1", "12"); + Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", params); + + SearchRequestBuilder builder = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(scriptedMetric("scripted").params(params).mapScript(mapScript)); + + SearchPhaseExecutionException ex = expectThrows(SearchPhaseExecutionException.class, builder::get); + assertThat(ex.getCause().getMessage(), containsString("Parameter name \"param1\" used in both aggregation and script parameters")); + } +} From 9f0b28bc9db1ed3b120edde227ab5ff12c47cf32 Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Tue, 17 Jul 2018 23:53:24 -0700 Subject: [PATCH 05/10] Disable deprecated params._agg/_aggs in docs integration tests and revise stored scripts to use state/states instead --- docs/build.gradle | 11 +++++++---- .../metrics/scripted-metric-aggregation.asciidoc | 2 -- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/build.gradle b/docs/build.gradle index 829db4381b046..36b620fc1ed4c 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -37,6 +37,9 @@ integTestCluster { extraConfigFile 'hunspell/en_US/en_US.dic', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic' // Whitelist reindexing from the local node so we can test it. setting 'reindex.remote.whitelist', '127.0.0.1:*' + + // TODO: remove this deprecation compatibility setting for 7.0 + systemProperty 'es.aggregations.enableDeprecatedScriptedMetricAggParam', 'false' } // remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed @@ -396,25 +399,25 @@ buildRestTests.setups['stored_scripted_metric_script'] = ''' - do: put_script: id: "my_init_script" - body: { "script": { "lang": "painless", "source": "params._agg.transactions = []" } } + body: { "script": { "lang": "painless", "source": "state.transactions = []" } } - match: { acknowledged: true } - do: put_script: id: "my_map_script" - body: { "script": { "lang": "painless", "source": "params._agg.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)" } } + body: { "script": { "lang": "painless", "source": "state.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)" } } - match: { acknowledged: true } - do: put_script: id: "my_combine_script" - body: { "script": { "lang": "painless", "source": "double profit = 0;for (t in params._agg.transactions) { profit += t; } return profit" } } + body: { "script": { "lang": "painless", "source": "double profit = 0;for (t in state.transactions) { profit += t; } return profit" } } - match: { acknowledged: true } - do: put_script: id: "my_reduce_script" - body: { "script": { "lang": "painless", "source": "double profit = 0;for (a in params._aggs) { profit += a; } return profit" } } + body: { "script": { "lang": "painless", "source": "double profit = 0;for (a in states) { profit += a; } return profit" } } - match: { acknowledged: true } ''' diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc index 7ec775a5891dd..c4857699f9805 100644 --- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc @@ -26,7 +26,6 @@ POST ledger/_search?size=0 -------------------------------------------------- // CONSOLE // TEST[setup:ledger] -// TEST[warning:params._agg/_aggs for scripted metric aggregations are deprecated, use state/states (not in params) instead. Set system property es.aggregations.enableDeprecatedScriptedMetricAggParam = false to disable this deprecated behavior.] <1> `map_script` is the only required parameter @@ -80,7 +79,6 @@ POST ledger/_search?size=0 -------------------------------------------------- // CONSOLE // TEST[setup:ledger,stored_scripted_metric_script] -// TEST[warning:params._agg/_aggs for scripted metric aggregations are deprecated, use state/states (not in params) instead. Set system property es.aggregations.enableDeprecatedScriptedMetricAggParam = false to disable this deprecated behavior.] <1> script parameters for `init`, `map` and `combine` scripts must be specified in a global `params` object so that it can be shared between the scripts. From bf13bf4b3779acc7d55e9658d0b9223a926d95e8 Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Wed, 18 Jul 2018 18:52:41 -0700 Subject: [PATCH 06/10] Revert unnecessary migrations doc change A relevant note should be added in the changes destined for 7.0; this PR is going to be backported to 6.x. --- docs/reference/migration/migrate_7_0/aggregations.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/aggregations.asciidoc b/docs/reference/migration/migrate_7_0/aggregations.asciidoc index 8cc1f81ff4764..b4f29935be9ab 100644 --- a/docs/reference/migration/migrate_7_0/aggregations.asciidoc +++ b/docs/reference/migration/migrate_7_0/aggregations.asciidoc @@ -22,5 +22,4 @@ The object used to share aggregation state between the scripts in a Scripted Met Aggregation is now a variable called `state` available in the script context, rather than being provided via the `params` object as `params._agg`. -The old `params._agg` variable is still available as well. This can be disabled by setting -the `es.aggregations.enableDeprecatedScriptedMetricAggParam` system property to `false`. +The old `params._agg` variable is still available as well. From 968173802f191689823289d017ba61754bf7d203 Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Thu, 19 Jul 2018 23:08:26 -0700 Subject: [PATCH 07/10] Replace deprecated _agg param bwc integration test with a couple of unit tests --- server/build.gradle | 25 +- .../ScriptedMetricAggParamsV6CompatIT.java | 1022 ----------------- ...alScriptedMetricAggStateV6CompatTests.java | 109 ++ ...MetricAggregatorAggStateV6CompatTests.java | 183 +++ .../ScriptedMetricAggregatorTests.java | 4 +- 5 files changed, 304 insertions(+), 1039 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggParamsV6CompatIT.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java diff --git a/server/build.gradle b/server/build.gradle index 6e37444734a9a..6a53c75444d76 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -334,22 +334,7 @@ if (isEclipse == false || project.path == ":server-tests") { classpath = project.test.classpath testClassesDirs = project.test.testClassesDirs include '**/*IT.class' - // TODO: remove ScriptedMetricAggParamsV6CompatIT for 7.0 - exclude '**/ScriptedMetricAggParamsV6CompatIT.class' } - - // TODO: remove this task dedicated to ScriptedMetricAggParamsV6CompatIT for 7.0 - task scriptedMetricAggDeprecatedParamIntegTest(type: RandomizedTestingTask, - description: 'Dedicated backwards compatibility integration test', - dependsOn: integTest.dependsOn.collect()) { - configure(BuildPlugin.commonTestConfig(project)) - classpath = integTest.classpath - testClassesDirs = integTest.testClassesDirs - systemProperty 'es.aggregations.enableDeprecatedScriptedMetricAggParam', 'true' - include '**/ScriptedMetricAggParamsV6CompatIT.class' - } - integTest.dependsOn scriptedMetricAggDeprecatedParamIntegTest - check.dependsOn integTest integTest.mustRunAfter test } @@ -359,7 +344,17 @@ additionalTest('testScriptDocValuesMissingV6Behaviour'){ include '**/ScriptDocValuesMissingV6BehaviourTests.class' systemProperty 'es.scripting.exception_for_missing_value', 'false' } + +// TODO: remove these compatibility tests in 7.0 +additionalTest('testScriptedMetricAggParamsV6Compatibility') { + include '**/ScriptedMetricAggregatorAggStateV6CompatTests.class' + include '**/InternalScriptedMetricAggStateV6CompatTests.class' + systemProperty 'es.aggregations.enableDeprecatedScriptedMetricAggParam', 'true' +} + test { // these are tested explicitly in separate test tasks exclude '**/*ScriptDocValuesMissingV6BehaviourTests.class' + exclude '**/ScriptedMetricAggregatorAggStateV6CompatTests.class' + exclude '**/InternalScriptedMetricAggStateV6CompatTests.class' } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggParamsV6CompatIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggParamsV6CompatIT.java deleted file mode 100644 index 73ad84036ae0c..0000000000000 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggParamsV6CompatIT.java +++ /dev/null @@ -1,1022 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.metrics; - -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.MockScriptPlugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.bucket.global.Global; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; -import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetric; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Before; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; -import java.util.function.Function; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.global; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.scriptedMetric; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.sameInstance; - -// This is a clone of relevant tests from ScriptedMetricIT from before they were converted away from the deprecated -// params._agg/_aggs state storage to the 'state' context variable. This test runs with the deprecated behavior enabled -// to ensure that it doesn't break. -// TODO: Remove this class for 7.0 when the deprecated behavior is removed. -@ClusterScope(scope = Scope.SUITE) -@ESIntegTestCase.SuiteScopeTestCase -public class ScriptedMetricAggParamsV6CompatIT extends ESIntegTestCase { - - private static long numDocs; - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(CustomScriptPlugin.class); - } - - public static class CustomScriptPlugin extends MockScriptPlugin { - - @Override - @SuppressWarnings("unchecked") - protected Map, Object>> pluginScripts() { - Map, Object>> scripts = new HashMap<>(); - - scripts.put("_agg['count'] = 1", vars -> - aggScript(vars, agg -> ((Map) agg).put("count", 1))); - - scripts.put("_agg.add(1)", vars -> - aggScript(vars, agg -> ((List) agg).add(1))); - - scripts.put("_agg[param1] = param2", vars -> - aggScript(vars, agg -> ((Map) agg).put(XContentMapValues.extractValue("params.param1", vars), - XContentMapValues.extractValue("params.param2", vars)))); - - scripts.put("vars.multiplier = 3", vars -> - ((Map) vars.get("vars")).put("multiplier", 3)); - - scripts.put("_agg.add(vars.multiplier)", vars -> - aggScript(vars, agg -> ((List) agg).add(XContentMapValues.extractValue("vars.multiplier", vars)))); - - // Equivalent to: - // - // newaggregation = []; - // sum = 0; - // - // for (a in _agg) { - // sum += a - // }; - // - // newaggregation.add(sum); - // return newaggregation" - // - scripts.put("sum agg values as a new aggregation", vars -> { - List newAggregation = new ArrayList(); - List agg = (List) vars.get("_agg"); - - if (agg != null) { - Integer sum = 0; - for (Object a : (List) agg) { - sum += ((Number) a).intValue(); - } - newAggregation.add(sum); - } - return newAggregation; - }); - - // Equivalent to: - // - // newaggregation = []; - // sum = 0; - // - // for (aggregation in _aggs) { - // for (a in aggregation) { - // sum += a - // } - // }; - // - // newaggregation.add(sum); - // return newaggregation" - // - scripts.put("sum aggs of agg values as a new aggregation", vars -> { - List newAggregation = new ArrayList(); - Integer sum = 0; - - List aggs = (List) vars.get("_aggs"); - for (Object aggregation : (List) aggs) { - if (aggregation != null) { - for (Object a : (List) aggregation) { - sum += ((Number) a).intValue(); - } - } - } - newAggregation.add(sum); - return newAggregation; - }); - - // Equivalent to: - // - // newaggregation = []; - // sum = 0; - // - // for (aggregation in _aggs) { - // for (a in aggregation) { - // sum += a - // } - // }; - // - // newaggregation.add(sum * multiplier); - // return newaggregation" - // - scripts.put("multiplied sum aggs of agg values as a new aggregation", vars -> { - Integer multiplier = (Integer) vars.get("multiplier"); - List newAggregation = new ArrayList(); - Integer sum = 0; - - List aggs = (List) vars.get("_aggs"); - for (Object aggregation : (List) aggs) { - if (aggregation != null) { - for (Object a : (List) aggregation) { - sum += ((Number) a).intValue(); - } - } - } - newAggregation.add(sum * multiplier); - return newAggregation; - }); - - return scripts; - } - - @SuppressWarnings("unchecked") - private static Object aggScript(Map vars, Consumer fn) { - T aggState = (T) vars.get("_agg"); - fn.accept(aggState); - return aggState; - } - } - - @Override - public void setupSuiteScopeCluster() throws Exception { - createIndex("idx"); - - List builders = new ArrayList<>(); - - numDocs = randomIntBetween(10, 100); - for (int i = 0; i < numDocs; i++) { - builders.add(client().prepareIndex("idx", "type", "" + i).setSource( - jsonBuilder().startObject().field("value", randomAlphaOfLengthBetween(5, 15)) - .field("l_value", i).endObject())); - } - indexRandom(true, builders); - - // creating an index to test the empty buckets functionality. The way it - // works is by indexing - // two docs {value: 0} and {value : 2}, then building a histogram agg - // with interval 1 and with empty - // buckets computed.. the empty bucket is the one associated with key - // "1". then each test will have - // to check that this bucket exists with the appropriate sub - // aggregations. - prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer").execute().actionGet(); - builders = new ArrayList<>(); - for (int i = 0; i < 2; i++) { - builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + i).setSource( - jsonBuilder().startObject().field("value", i * 2).endObject())); - } - - // When using the MockScriptPlugin we can map Stored scripts to inline scripts: - // the id of the stored script is used in test method while the source of the stored script - // must match a predefined script from CustomScriptPlugin.pluginScripts() method - assertAcked(client().admin().cluster().preparePutStoredScript() - .setId("initScript_stored") - .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + - " \"source\": \"vars.multiplier = 3\"} }"), XContentType.JSON)); - - assertAcked(client().admin().cluster().preparePutStoredScript() - .setId("mapScript_stored") - .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + - " \"source\": \"_agg.add(vars.multiplier)\"} }"), XContentType.JSON)); - - assertAcked(client().admin().cluster().preparePutStoredScript() - .setId("combineScript_stored") - .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + - " \"source\": \"sum agg values as a new aggregation\"} }"), XContentType.JSON)); - - assertAcked(client().admin().cluster().preparePutStoredScript() - .setId("reduceScript_stored") - .setContent(new BytesArray("{\"script\": {\"lang\": \"" + MockScriptPlugin.NAME + "\"," + - " \"source\": \"sum aggs of agg values as a new aggregation\"} }"), XContentType.JSON)); - - indexRandom(true, builders); - ensureSearchable(); - } - - private Path config; - - @Before - public void setUp() throws Exception { - super.setUp(); - config = createTempDir().resolve("config"); - final Path scripts = config.resolve("scripts"); - - try { - Files.createDirectories(scripts); - // When using the MockScriptPlugin we can map File scripts to inline scripts: - // the name of the file script is used in test method while the source of the file script - // must match a predefined script from CustomScriptPlugin.pluginScripts() method - Files.write(scripts.resolve("init_script.mockscript"), "vars.multiplier = 3".getBytes("UTF-8")); - Files.write(scripts.resolve("map_script.mockscript"), "_agg.add(vars.multiplier)".getBytes("UTF-8")); - Files.write(scripts.resolve("combine_script.mockscript"), "sum agg values as a new aggregation".getBytes("UTF-8")); - Files.write(scripts.resolve("reduce_script.mockscript"), "sum aggs of agg values as a new aggregation".getBytes("UTF-8")); - } catch (IOException e) { - throw new RuntimeException("failed to create scripts"); - } - } - - @Override - protected Path nodeConfigPath(int nodeOrdinal) { - return config; - } - - public void testMap() { - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg['count'] = 1", Collections.emptyMap()); - - SearchResponse response = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(scriptedMetric("scripted").mapScript(mapScript)) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); - int numShardsRun = 0; - for (Object object : aggregationList) { - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Map.class)); - Map map = (Map) object; - assertThat(map.size(), lessThanOrEqualTo(1)); - if (map.size() == 1) { - assertThat(map.get("count"), notNullValue()); - assertThat(map.get("count"), instanceOf(Number.class)); - assertThat(map.get("count"), equalTo((Number) 1)); - numShardsRun++; - } - } - // We don't know how many shards will have documents but we need to make - // sure that at least one shard ran the map script - assertThat(numShardsRun, greaterThan(0)); - } - - public void testExplicitAggParam() { - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", Collections.emptyMap()); - - SearchResponse response = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(scriptedMetric("scripted").params(params).mapScript(mapScript)) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); - long totalCount = 0; - for (Object object : aggregationList) { - assertThat(object, notNullValue()); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - for (Object o : list) { - assertThat(o, notNullValue()); - assertThat(o, instanceOf(Number.class)); - Number numberValue = (Number) o; - assertThat(numberValue, equalTo((Number) 1)); - totalCount += numberValue.longValue(); - } - } - assertThat(totalCount, equalTo(numDocs)); - } - - public void testMapWithParamsAndImplicitAggMap() { - // Split the params up between the script and the aggregation. - // Don't put any _agg map in params. - Map scriptParams = Collections.singletonMap("param1", "12"); - Map aggregationParams = Collections.singletonMap("param2", 1); - - // The _agg hashmap will be available even if not declared in the params map - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg[param1] = param2", scriptParams); - - SearchResponse response = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(scriptedMetric("scripted").params(aggregationParams).mapScript(mapScript)) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); - int numShardsRun = 0; - for (Object object : aggregationList) { - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Map.class)); - Map map = (Map) object; - for (Map.Entry entry : map.entrySet()) { - assertThat(entry, notNullValue()); - assertThat(entry.getKey(), notNullValue()); - assertThat(entry.getKey(), instanceOf(String.class)); - assertThat(entry.getValue(), notNullValue()); - assertThat(entry.getValue(), instanceOf(Number.class)); - String stringValue = (String) entry.getKey(); - assertThat(stringValue, equalTo("12")); - Number numberValue = (Number) entry.getValue(); - assertThat(numberValue, equalTo((Number) 1)); - numShardsRun++; - } - } - assertThat(numShardsRun, greaterThan(0)); - } - - public void testInitMapWithParams() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .params(params) - .initScript( - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap())) - .mapScript(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, - "_agg.add(vars.multiplier)", Collections.emptyMap()))) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); - long totalCount = 0; - for (Object object : aggregationList) { - assertThat(object, notNullValue()); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - for (Object o : list) { - assertThat(o, notNullValue()); - assertThat(o, instanceOf(Number.class)); - Number numberValue = (Number) o; - assertThat(numberValue, equalTo((Number) 3)); - totalCount += numberValue.longValue(); - } - } - assertThat(totalCount, equalTo(numDocs * 3)); - } - - public void testMapCombineWithParams() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", Collections.emptyMap()); - Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .params(params) - .mapScript(mapScript) - .combineScript(combineScript)) - .execute().actionGet(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); - long totalCount = 0; - for (Object object : aggregationList) { - assertThat(object, notNullValue()); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - for (Object o : list) { - assertThat(o, notNullValue()); - assertThat(o, instanceOf(Number.class)); - Number numberValue = (Number) o; - // A particular shard may not have any documents stored on it so - // we have to assume the lower bound may be 0. The check at the - // bottom of the test method will make sure the count is correct - assertThat(numberValue.longValue(), allOf(greaterThanOrEqualTo(0L), lessThanOrEqualTo(numDocs))); - totalCount += numberValue.longValue(); - } - } - assertThat(totalCount, equalTo(numDocs)); - } - - public void testInitMapCombineWithParams() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .params(params) - .initScript(initScript) - .mapScript(mapScript) - .combineScript(combineScript)) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(getNumShards("idx").numPrimaries)); - long totalCount = 0; - for (Object object : aggregationList) { - assertThat(object, notNullValue()); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - for (Object o : list) { - assertThat(o, notNullValue()); - assertThat(o, instanceOf(Number.class)); - Number numberValue = (Number) o; - // A particular shard may not have any documents stored on it so - // we have to assume the lower bound may be 0. The check at the - // bottom of the test method will make sure the count is correct - assertThat(numberValue.longValue(), allOf(greaterThanOrEqualTo(0L), lessThanOrEqualTo(numDocs * 3))); - totalCount += numberValue.longValue(); - } - } - assertThat(totalCount, equalTo(numDocs * 3)); - } - - public void testInitMapCombineReduceWithParams() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .params(params) - .initScript(initScript) - .mapScript(mapScript) - .combineScript(combineScript) - .reduceScript(reduceScript)) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(1)); - Object object = aggregationList.get(0); - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Number.class)); - assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); - } - - @SuppressWarnings("rawtypes") - public void testInitMapCombineReduceGetProperty() throws Exception { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); - - SearchResponse searchResponse = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - global("global") - .subAggregation( - scriptedMetric("scripted") - .params(params) - .initScript(initScript) - .mapScript(mapScript) - .combineScript(combineScript) - .reduceScript(reduceScript))) - .get(); - - assertSearchResponse(searchResponse); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocs)); - - Global global = searchResponse.getAggregations().get("global"); - assertThat(global, notNullValue()); - assertThat(global.getName(), equalTo("global")); - assertThat(global.getDocCount(), equalTo(numDocs)); - assertThat(global.getAggregations(), notNullValue()); - assertThat(global.getAggregations().asMap().size(), equalTo(1)); - - ScriptedMetric scriptedMetricAggregation = global.getAggregations().get("scripted"); - assertThat(scriptedMetricAggregation, notNullValue()); - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(1)); - Object object = aggregationList.get(0); - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Number.class)); - assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); - assertThat(((InternalAggregation)global).getProperty("scripted"), sameInstance(scriptedMetricAggregation)); - assertThat((List) ((InternalAggregation)global).getProperty("scripted.value"), sameInstance((List) aggregationList)); - assertThat((List) ((InternalAggregation)scriptedMetricAggregation).getProperty("value"), sameInstance((List) aggregationList)); - } - - public void testMapCombineReduceWithParams() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .params(params) - .mapScript(mapScript) - .combineScript(combineScript) - .reduceScript(reduceScript)) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(1)); - Object object = aggregationList.get(0); - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Number.class)); - assertThat(((Number) object).longValue(), equalTo(numDocs)); - } - - public void testInitMapReduceWithParams() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .params(params) - .initScript(initScript) - .mapScript(mapScript) - .reduceScript(reduceScript)) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(1)); - Object object = aggregationList.get(0); - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Number.class)); - assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); - } - - public void testMapReduceWithParams() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .params(params) - .mapScript(mapScript) - .reduceScript(reduceScript)) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(1)); - Object object = aggregationList.get(0); - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Number.class)); - assertThat(((Number) object).longValue(), equalTo(numDocs)); - } - - public void testInitMapCombineReduceWithParamsAndReduceParams() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Map reduceParams = new HashMap<>(); - reduceParams.put("multiplier", 4); - - Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "multiplied sum aggs of agg values as a new aggregation", reduceParams); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .params(params) - .initScript(initScript) - .mapScript(mapScript) - .combineScript(combineScript) - .reduceScript(reduceScript)) - .execute().actionGet(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(1)); - Object object = aggregationList.get(0); - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Number.class)); - assertThat(((Number) object).longValue(), equalTo(numDocs * 12)); - } - - public void testInitMapCombineReduceWithParamsStored() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - scriptedMetric("scripted") - .params(params) - .initScript( - new Script(ScriptType.STORED, null, "initScript_stored", Collections.emptyMap())) - .mapScript( - new Script(ScriptType.STORED, null, "mapScript_stored", Collections.emptyMap())) - .combineScript( - new Script(ScriptType.STORED, null, "combineScript_stored", Collections.emptyMap())) - .reduceScript( - new Script(ScriptType.STORED, null, "reduceScript_stored", Collections.emptyMap()))) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - - Aggregation aggregation = response.getAggregations().get("scripted"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) aggregation; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(1)); - Object object = aggregationList.get(0); - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Number.class)); - assertThat(((Number) object).longValue(), equalTo(numDocs * 3)); - } - - public void testInitMapCombineReduceWithParamsAsSubAgg() { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); - - SearchResponse response = client() - .prepareSearch("idx") - .setQuery(matchAllQuery()).setSize(1000) - .addAggregation( - histogram("histo") - .field("l_value") - .interval(1) - .subAggregation( - scriptedMetric("scripted") - .params(params) - .initScript(initScript) - .mapScript(mapScript) - .combineScript(combineScript) - .reduceScript(reduceScript))) - .get(); - assertSearchResponse(response); - assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); - Aggregation aggregation = response.getAggregations().get("histo"); - assertThat(aggregation, notNullValue()); - assertThat(aggregation, instanceOf(Histogram.class)); - Histogram histoAgg = (Histogram) aggregation; - assertThat(histoAgg.getName(), equalTo("histo")); - List buckets = histoAgg.getBuckets(); - assertThat(buckets, notNullValue()); - for (Bucket b : buckets) { - assertThat(b, notNullValue()); - assertThat(b.getDocCount(), equalTo(1L)); - Aggregations subAggs = b.getAggregations(); - assertThat(subAggs, notNullValue()); - assertThat(subAggs.asList().size(), equalTo(1)); - Aggregation subAgg = subAggs.get("scripted"); - assertThat(subAgg, notNullValue()); - assertThat(subAgg, instanceOf(ScriptedMetric.class)); - ScriptedMetric scriptedMetricAggregation = (ScriptedMetric) subAgg; - assertThat(scriptedMetricAggregation.getName(), equalTo("scripted")); - assertThat(scriptedMetricAggregation.aggregation(), notNullValue()); - assertThat(scriptedMetricAggregation.aggregation(), instanceOf(ArrayList.class)); - List aggregationList = (List) scriptedMetricAggregation.aggregation(); - assertThat(aggregationList.size(), equalTo(1)); - Object object = aggregationList.get(0); - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Number.class)); - assertThat(((Number) object).longValue(), equalTo(3L)); - } - } - - public void testEmptyAggregation() throws Exception { - Map varsMap = new HashMap<>(); - varsMap.put("multiplier", 1); - - Map params = new HashMap<>(); - params.put("_agg", new ArrayList<>()); - params.put("vars", varsMap); - - Script initScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "vars.multiplier = 3", Collections.emptyMap()); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(vars.multiplier)", Collections.emptyMap()); - Script combineScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum agg values as a new aggregation", Collections.emptyMap()); - Script reduceScript = - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "sum aggs of agg values as a new aggregation", Collections.emptyMap()); - - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0) - .subAggregation( - scriptedMetric("scripted") - .params(params) - .initScript(initScript) - .mapScript(mapScript) - .combineScript(combineScript) - .reduceScript(reduceScript))) - .get(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L)); - Histogram histo = searchResponse.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - Histogram.Bucket bucket = histo.getBuckets().get(1); - assertThat(bucket, notNullValue()); - - ScriptedMetric scriptedMetric = bucket.getAggregations().get("scripted"); - assertThat(scriptedMetric, notNullValue()); - assertThat(scriptedMetric.getName(), equalTo("scripted")); - assertThat(scriptedMetric.aggregation(), notNullValue()); - assertThat(scriptedMetric.aggregation(), instanceOf(List.class)); - @SuppressWarnings("unchecked") // We'll just get a ClassCastException a couple lines down if we're wrong, its ok. - List aggregationResult = (List) scriptedMetric.aggregation(); - assertThat(aggregationResult.size(), equalTo(1)); - assertThat(aggregationResult.get(0), equalTo(0)); - } - - /** - * Make sure that a request using a script does not get cached and a request - * not using a script does get cached. - */ - public void testDontCacheScripts() throws Exception { - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg['count'] = 1", Collections.emptyMap()); - assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") - .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) - .get()); - indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), - client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); - - // Make sure we are starting with a clear cache - assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() - .getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() - .getMissCount(), equalTo(0L)); - - // Test that a request using a script does not get cached - SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) - .addAggregation(scriptedMetric("foo").mapScript(mapScript)).get(); - assertSearchResponse(r); - - assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() - .getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() - .getMissCount(), equalTo(0L)); - } - - public void testConflictingAggAndScriptParams() { - Map params = Collections.singletonMap("param1", "12"); - Script mapScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_agg.add(1)", params); - - SearchRequestBuilder builder = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(scriptedMetric("scripted").params(params).mapScript(mapScript)); - - SearchPhaseExecutionException ex = expectThrows(SearchPhaseExecutionException.class, builder::get); - assertThat(ex.getCause().getMessage(), containsString("Parameter name \"param1\" used in both aggregation and script parameters")); - } -} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java new file mode 100644 index 0000000000000..218c6cde42689 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.scripted; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.MockScriptEngine; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptedMetricAggContexts; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.aggregations.Aggregation.CommonFields; +import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.test.InternalAggregationTestCase; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Predicate; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.sameInstance; + +/** + * This test verifies that the _aggs param is added correctly when the system property + * "es.aggregations.enableDeprecatedScriptedMetricAggParam" is set to true. + */ +public class InternalScriptedMetricAggStateV6CompatTests extends InternalAggregationTestCase { + + private static final String REDUCE_SCRIPT_NAME = "reduceScript"; + + @Override + protected InternalScriptedMetric createTestInstance(String name, List pipelineAggregators, + Map metaData) { + Script reduceScript = new Script(ScriptType.INLINE, MockScriptEngine.NAME, REDUCE_SCRIPT_NAME, Collections.emptyMap()); + return new InternalScriptedMetric(name, "agg value", reduceScript, pipelineAggregators, metaData); + } + + /** + * Mock of the script service. The script that is run looks at the + * "_aggs" parameter to verify that it was put in place by InternalScriptedMetric. + */ + @Override + protected ScriptService mockScriptService() { + Function, Object> script = params -> { + Object aggs = params.get("_aggs"); + Object states = params.get("states"); + assertThat(aggs, instanceOf(List.class)); + assertThat(aggs, sameInstance(states)); + return aggs; + }; + + @SuppressWarnings("unchecked") + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, + Collections.singletonMap(REDUCE_SCRIPT_NAME, script)); + Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + } + + @Override + protected void assertReduced(InternalScriptedMetric reduced, List inputs) { + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); + } + + @Override + protected Reader instanceReader() { + return InternalScriptedMetric::new; + } + + @Override + protected void assertFromXContent(InternalScriptedMetric aggregation, ParsedAggregation parsedAggregation) {} + + @Override + protected Predicate excludePathsFromXContentInsertion() { + return path -> path.contains(CommonFields.VALUE.getPreferredName()); + } + + @Override + protected InternalScriptedMetric mutateInstance(InternalScriptedMetric instance) { + String name = instance.getName(); + Object value = instance.aggregation(); + Script reduceScript = instance.reduceScript; + List pipelineAggregators = instance.pipelineAggregators(); + Map metaData = instance.getMetaData(); + return new InternalScriptedMetric(name + randomAlphaOfLength(5), value, reduceScript, pipelineAggregators, + metaData); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java new file mode 100644 index 0000000000000..5f7744d2f2a9f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java @@ -0,0 +1,183 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.scripted; + +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.MockScriptEngine; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptedMetricAggContexts; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import static java.util.Collections.singleton; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.sameInstance; + +/** + * This test verifies that the _agg param is added correctly when the system property + * "es.aggregations.enableDeprecatedScriptedMetricAggParam" is set to true. + */ +public class ScriptedMetricAggregatorAggStateV6CompatTests extends AggregatorTestCase { + + private static final String AGG_NAME = "scriptedMetric"; + private static final Script INIT_SCRIPT = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "initScript", Collections.emptyMap()); + private static final Script MAP_SCRIPT = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "mapScript", Collections.emptyMap()); + private static final Script COMBINE_SCRIPT = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "combineScript", + Collections.emptyMap()); + + private static final Script INIT_SCRIPT_EXPLICIT_AGG = new Script(ScriptType.INLINE, MockScriptEngine.NAME, + "initScriptExplicitAgg", Collections.emptyMap()); + private static final Script MAP_SCRIPT_EXPLICIT_AGG = new Script(ScriptType.INLINE, MockScriptEngine.NAME, + "mapScriptExplicitAgg", Collections.emptyMap()); + private static final Script COMBINE_SCRIPT_EXPLICIT_AGG = new Script(ScriptType.INLINE, MockScriptEngine.NAME, + "combineScriptExplicitAgg", Collections.emptyMap()); + private static final String EXPLICIT_AGG_OBJECT = "Explicit agg object"; + + private static final Map, Object>> SCRIPTS = new HashMap<>(); + + @BeforeClass + @SuppressWarnings("unchecked") + public static void initMockScripts() { + // If _agg is provided implicitly, it should be the same objects as "state" from the context. + SCRIPTS.put("initScript", params -> { + Object agg = params.get("_agg"); + Object state = params.get("state"); + assertThat(agg, instanceOf(Map.class)); + assertThat(agg, sameInstance(state)); + return agg; + }); + SCRIPTS.put("mapScript", params -> { + Object agg = params.get("_agg"); + Object state = params.get("state"); + assertThat(agg, instanceOf(Map.class)); + assertThat(agg, sameInstance(state)); + return agg; + }); + SCRIPTS.put("combineScript", params -> { + Object agg = params.get("_agg"); + Object state = params.get("state"); + assertThat(agg, instanceOf(Map.class)); + assertThat(agg, sameInstance(state)); + return agg; + }); + + SCRIPTS.put("initScriptExplicitAgg", params -> { + Object agg = params.get("_agg"); + assertThat(agg, equalTo(EXPLICIT_AGG_OBJECT)); + return agg; + }); + SCRIPTS.put("mapScriptExplicitAgg", params -> { + Object agg = params.get("_agg"); + assertThat(agg, equalTo(EXPLICIT_AGG_OBJECT)); + return agg; + }); + SCRIPTS.put("combineScriptExplicitAgg", params -> { + Object agg = params.get("_agg"); + assertThat(agg, equalTo(EXPLICIT_AGG_OBJECT)); + return agg; + }); + } + + /** + * Test that the _agg param is implicitly added + */ + public void testWithImplicitAggParam() throws IOException { + try (Directory directory = newDirectory()) { + Integer numDocs = 10; + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + for (int i = 0; i < numDocs; i++) { + indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i))); + } + } + try (IndexReader indexReader = DirectoryReader.open(directory)) { + ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); + aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT).combineScript(COMBINE_SCRIPT); + search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + } + } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); + } + + /** + * Test that an explicitly added _agg param is honored + */ + public void testWithExplicitAggParam() throws IOException { + try (Directory directory = newDirectory()) { + Integer numDocs = 10; + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + for (int i = 0; i < numDocs; i++) { + indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i))); + } + } + + Map aggParams = new HashMap<>(); + aggParams.put("_agg", EXPLICIT_AGG_OBJECT); + + try (IndexReader indexReader = DirectoryReader.open(directory)) { + ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); + aggregationBuilder + .params(aggParams) + .initScript(INIT_SCRIPT_EXPLICIT_AGG) + .mapScript(MAP_SCRIPT_EXPLICIT_AGG) + .combineScript(COMBINE_SCRIPT_EXPLICIT_AGG); + search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + } + } + + assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING); + } + + /** + * We cannot use Mockito for mocking QueryShardContext in this case because + * script-related methods (e.g. QueryShardContext#getLazyExecutableScript) + * is final and cannot be mocked + */ + @Override + protected QueryShardContext queryShardContextMock(MapperService mapperService, final MappedFieldType[] fieldTypes, + CircuitBreakerService circuitBreakerService) { + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS); + Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); + ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new QueryShardContext(0, mapperService.getIndexSettings(), null, null, mapperService, null, scriptService, + xContentRegistry(), writableRegistry(), null, null, System::currentTimeMillis, null); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java index a9ef4a91ea32a..2727ecccf8c0d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java @@ -88,7 +88,7 @@ public static void initMockScripts() { Map state = (Map) params.get("state"); state.put("collector", new ArrayList()); return state; - }); + }); SCRIPTS.put("mapScript", params -> { Map state = (Map) params.get("state"); ((List) state.get("collector")).add(1); // just add 1 for each doc the script is run on @@ -103,7 +103,7 @@ public static void initMockScripts() { Map state = (Map) params.get("state"); state.put("collector", new ArrayList()); return state; - }); + }); SCRIPTS.put("mapScriptScore", params -> { Map state = (Map) params.get("state"); ((List) state.get("collector")).add(((Number) params.get("_score")).doubleValue()); From 20ac8d8045008c5971e718d84749b3629218f297 Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Thu, 19 Jul 2018 23:24:59 -0700 Subject: [PATCH 08/10] Fix compatibility test after merge --- .../ScriptedMetricAggregatorAggStateV6CompatTests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java index 5f7744d2f2a9f..1c4b5387170e9 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java @@ -26,10 +26,8 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptedMetricAggContexts; @@ -172,8 +170,7 @@ public void testWithExplicitAggParam() throws IOException { * is final and cannot be mocked */ @Override - protected QueryShardContext queryShardContextMock(MapperService mapperService, final MappedFieldType[] fieldTypes, - CircuitBreakerService circuitBreakerService) { + protected QueryShardContext queryShardContextMock(MapperService mapperService) { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); From 3ea60eeaf62573406230188e0a533f3a659396e9 Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Wed, 1 Aug 2018 21:19:45 -0700 Subject: [PATCH 09/10] Rename backwards compatibility system property per code review feedback --- .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 2 +- docs/build.gradle | 2 +- server/build.gradle | 2 +- .../org/elasticsearch/script/ScriptedMetricAggContexts.java | 6 +++--- .../InternalScriptedMetricAggStateV6CompatTests.java | 2 +- .../ScriptedMetricAggregatorAggStateV6CompatTests.java | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 8afde015828d5..eaf7b2e4a3f6e 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -754,7 +754,7 @@ class BuildPlugin implements Plugin { systemProperty 'tests.security.manager', 'true' systemProperty 'jna.nosys', 'true' // TODO: remove this deprecation compatibility setting for 7.0 - systemProperty 'es.aggregations.enableDeprecatedScriptedMetricAggParam', 'false' + systemProperty 'es.aggregations.enable_scripted_metric_agg_param', 'false' // TODO: remove setting logging level via system property systemProperty 'tests.logger.level', 'WARN' for (Map.Entry property : System.properties.entrySet()) { diff --git a/docs/build.gradle b/docs/build.gradle index 36b620fc1ed4c..569b54c134ea1 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -39,7 +39,7 @@ integTestCluster { setting 'reindex.remote.whitelist', '127.0.0.1:*' // TODO: remove this deprecation compatibility setting for 7.0 - systemProperty 'es.aggregations.enableDeprecatedScriptedMetricAggParam', 'false' + systemProperty 'es.aggregations.enable_scripted_metric_agg_param', 'false' } // remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed diff --git a/server/build.gradle b/server/build.gradle index df54c9f21a691..ff4f05e64c564 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -343,7 +343,7 @@ if (isEclipse == false || project.path == ":server-tests") { additionalTest('testScriptedMetricAggParamsV6Compatibility') { include '**/ScriptedMetricAggregatorAggStateV6CompatTests.class' include '**/InternalScriptedMetricAggStateV6CompatTests.class' - systemProperty 'es.aggregations.enableDeprecatedScriptedMetricAggParam', 'true' + systemProperty 'es.aggregations.enable_scripted_metric_agg_param', 'true' } test { diff --git a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java index db08eceb502d9..0091f211416fa 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java @@ -39,14 +39,14 @@ public class ScriptedMetricAggContexts { // Public for access from tests public static final String AGG_PARAM_DEPRECATION_WARNING = "params._agg/_aggs for scripted metric aggregations are deprecated, use state/states (not in params) instead. " + - "Set system property es.aggregations.enableDeprecatedScriptedMetricAggParam = false to disable this deprecated behavior."; + "Set system property es.aggregations.enable_scripted_metric_agg_param = false to disable this deprecated behavior."; public static boolean deprecatedAggParamEnabled() { boolean enabled = Boolean.parseBoolean( - System.getProperty("es.aggregations.enableDeprecatedScriptedMetricAggParam", "true")); + System.getProperty("es.aggregations.enable_scripted_metric_agg_param", "true")); if (enabled) { - DEPRECATION_LOGGER.deprecatedAndMaybeLog("enableDeprecatedScriptedMetricAggParams", AGG_PARAM_DEPRECATION_WARNING); + DEPRECATION_LOGGER.deprecatedAndMaybeLog("enable_scripted_metric_agg_param", AGG_PARAM_DEPRECATION_WARNING); } return enabled; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java index 218c6cde42689..4abf68a960b11 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java @@ -44,7 +44,7 @@ /** * This test verifies that the _aggs param is added correctly when the system property - * "es.aggregations.enableDeprecatedScriptedMetricAggParam" is set to true. + * "es.aggregations.enable_scripted_metric_agg_param" is set to true. */ public class InternalScriptedMetricAggStateV6CompatTests extends InternalAggregationTestCase { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java index 1c4b5387170e9..bf78cae711b9d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java @@ -51,7 +51,7 @@ /** * This test verifies that the _agg param is added correctly when the system property - * "es.aggregations.enableDeprecatedScriptedMetricAggParam" is set to true. + * "es.aggregations.enable_scripted_metric_agg_param" is set to true. */ public class ScriptedMetricAggregatorAggStateV6CompatTests extends AggregatorTestCase { From 6009712989dd560ba007b0f1abcf381ee12095b2 Mon Sep 17 00:00:00 2001 From: Jonathan Little Date: Wed, 8 Aug 2018 20:50:04 -0700 Subject: [PATCH 10/10] Tweak deprecation warning text per review feedback --- .../org/elasticsearch/script/ScriptedMetricAggContexts.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java index 0091f211416fa..0c34c59b7be58 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java @@ -39,7 +39,7 @@ public class ScriptedMetricAggContexts { // Public for access from tests public static final String AGG_PARAM_DEPRECATION_WARNING = "params._agg/_aggs for scripted metric aggregations are deprecated, use state/states (not in params) instead. " + - "Set system property es.aggregations.enable_scripted_metric_agg_param = false to disable this deprecated behavior."; + "Use -Des.aggregations.enable_scripted_metric_agg_param=false to disable."; public static boolean deprecatedAggParamEnabled() { boolean enabled = Boolean.parseBoolean(