-
Notifications
You must be signed in to change notification settings - Fork 25.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[ML] Return statistics about forecasts as part of the jobsstats and usage API #31647
Merged
Merged
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
0b4bef4
add ForecastStats to job stats
3eceac9
implement stats collection of forecasts, extending the getjobstats api
11ff017
fix stylecheck issues
1b93000
add forecastStats Writable interface implementation with BWC layer
b10eb4c
change field to forecasted_jobs, always write it and fix chained merges
a0f6ed5
add a test for forecast by status for the usage endpoint
f9804dc
add documentation about forecast statistics shown as part of jobcounts.
d6dd365
fix NPE when search fails
d1fa2fc
fix checkstyle failure
d1ed68d
fix JobStatsMonitoringDocTests
5f91cbc
improve documentation
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
82 changes: 82 additions & 0 deletions
82
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/stats/CountAccumulator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.core.ml.stats; | ||
|
|
||
| import org.elasticsearch.common.io.stream.StreamInput; | ||
| import org.elasticsearch.common.io.stream.StreamOutput; | ||
| import org.elasticsearch.common.io.stream.Writeable; | ||
| import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.Map.Entry; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
|
|
||
| /** | ||
| * An accumulator for simple counts where statistical measures | ||
| * are not of interest. | ||
| */ | ||
| public class CountAccumulator implements Writeable { | ||
|
|
||
| private Map<String, Long> counts; | ||
|
|
||
| public CountAccumulator() { | ||
| this.counts = new HashMap<String, Long>(); | ||
| } | ||
|
|
||
| private CountAccumulator(Map<String, Long> counts) { | ||
| this.counts = counts; | ||
| } | ||
|
|
||
| public CountAccumulator(StreamInput in) throws IOException { | ||
| this.counts = in.readMap(StreamInput::readString, StreamInput::readLong); | ||
| } | ||
|
|
||
| public void merge(CountAccumulator other) { | ||
| counts = Stream.of(counts, other.counts).flatMap(m -> m.entrySet().stream()) | ||
| .collect(Collectors.toMap(Entry::getKey, Entry::getValue, (x, y) -> x + y)); | ||
| } | ||
|
|
||
| public void add(String key, Long count) { | ||
| counts.put(key, counts.getOrDefault(key, 0L) + count); | ||
| } | ||
|
|
||
| public Map<String, Long> asMap() { | ||
| return counts; | ||
| } | ||
|
|
||
| public static CountAccumulator fromTermsAggregation(StringTerms termsAggregation) { | ||
| return new CountAccumulator(termsAggregation.getBuckets().stream() | ||
| .collect(Collectors.toMap(bucket -> bucket.getKeyAsString(), bucket -> bucket.getDocCount()))); | ||
| } | ||
|
|
||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeMap(counts, StreamOutput::writeString, StreamOutput::writeLong); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(counts); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object obj) { | ||
| if (obj == null) { | ||
| return false; | ||
| } | ||
|
|
||
| if (getClass() != obj.getClass()) { | ||
| return false; | ||
| } | ||
|
|
||
| CountAccumulator other = (CountAccumulator) obj; | ||
| return Objects.equals(counts, other.counts); | ||
| } | ||
| } |
152 changes: 152 additions & 0 deletions
152
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/stats/ForecastStats.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.core.ml.stats; | ||
|
|
||
| import org.elasticsearch.common.io.stream.StreamInput; | ||
| import org.elasticsearch.common.io.stream.StreamOutput; | ||
| import org.elasticsearch.common.io.stream.Writeable; | ||
| import org.elasticsearch.common.xcontent.ToXContentObject; | ||
| import org.elasticsearch.common.xcontent.XContentBuilder; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
|
|
||
| /** | ||
| * A class to hold statistics about forecasts. | ||
| */ | ||
| public class ForecastStats implements ToXContentObject, Writeable { | ||
|
|
||
| public static class Fields { | ||
| public static final String TOTAL = "total"; | ||
| public static final String FORECASTED_JOBS = "forecasted_jobs"; | ||
| public static final String MEMORY = "memory_bytes"; | ||
| public static final String RUNTIME = "processing_time_ms"; | ||
| public static final String RECORDS = "records"; | ||
| public static final String STATUSES = "status"; | ||
| } | ||
|
|
||
| private long total; | ||
| private long forecastedJobs; | ||
| private StatsAccumulator memoryStats; | ||
| private StatsAccumulator recordStats; | ||
| private StatsAccumulator runtimeStats; | ||
| private CountAccumulator statusCounts; | ||
|
|
||
| public ForecastStats() { | ||
| this.total = 0; | ||
| this.forecastedJobs = 0; | ||
| this.memoryStats = new StatsAccumulator(); | ||
| this.recordStats = new StatsAccumulator(); | ||
| this.runtimeStats = new StatsAccumulator(); | ||
| this.statusCounts = new CountAccumulator(); | ||
| } | ||
|
|
||
| /* | ||
| * Construct ForecastStats for 1 job. Additional statistics can be added by merging other ForecastStats into it. | ||
| */ | ||
| public ForecastStats(long total, StatsAccumulator memoryStats, StatsAccumulator recordStats, StatsAccumulator runtimeStats, | ||
| CountAccumulator statusCounts) { | ||
| this.total = total; | ||
| this.forecastedJobs = total > 0 ? 1 : 0; | ||
| this.memoryStats = Objects.requireNonNull(memoryStats); | ||
| this.recordStats = Objects.requireNonNull(recordStats); | ||
| this.runtimeStats = Objects.requireNonNull(runtimeStats); | ||
| this.statusCounts = Objects.requireNonNull(statusCounts); | ||
| } | ||
|
|
||
| public ForecastStats(StreamInput in) throws IOException { | ||
| this.total = in.readLong(); | ||
| this.forecastedJobs = in.readLong(); | ||
| this.memoryStats = new StatsAccumulator(in); | ||
| this.recordStats = new StatsAccumulator(in); | ||
| this.runtimeStats = new StatsAccumulator(in); | ||
| this.statusCounts = new CountAccumulator(in); | ||
| } | ||
|
|
||
| public ForecastStats merge(ForecastStats other) { | ||
| if (other == null) { | ||
| return this; | ||
| } | ||
| total += other.total; | ||
| forecastedJobs += other.forecastedJobs; | ||
| memoryStats.merge(other.memoryStats); | ||
| recordStats.merge(other.recordStats); | ||
| runtimeStats.merge(other.runtimeStats); | ||
| statusCounts.merge(other.statusCounts); | ||
|
|
||
| return this; | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| builder.startObject(); | ||
| doXContentBody(builder, params); | ||
| return builder.endObject(); | ||
| } | ||
|
|
||
| public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { | ||
| builder.field(Fields.TOTAL, total); | ||
| builder.field(Fields.FORECASTED_JOBS, forecastedJobs); | ||
|
|
||
| if (total > 0) { | ||
| builder.field(Fields.MEMORY, memoryStats.asMap()); | ||
| builder.field(Fields.RECORDS, recordStats.asMap()); | ||
| builder.field(Fields.RUNTIME, runtimeStats.asMap()); | ||
| builder.field(Fields.STATUSES, statusCounts.asMap()); | ||
| } | ||
|
|
||
| return builder; | ||
| } | ||
|
|
||
| public Map<String, Object> asMap() { | ||
| Map<String, Object> map = new HashMap<>(); | ||
| map.put(Fields.TOTAL, total); | ||
| map.put(Fields.FORECASTED_JOBS, forecastedJobs); | ||
|
|
||
| if (total > 0) { | ||
| map.put(Fields.MEMORY, memoryStats.asMap()); | ||
| map.put(Fields.RECORDS, recordStats.asMap()); | ||
| map.put(Fields.RUNTIME, runtimeStats.asMap()); | ||
| map.put(Fields.STATUSES, statusCounts.asMap()); | ||
| } | ||
|
|
||
| return map; | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeLong(total); | ||
| out.writeLong(forecastedJobs); | ||
| memoryStats.writeTo(out); | ||
| recordStats.writeTo(out); | ||
| runtimeStats.writeTo(out); | ||
| statusCounts.writeTo(out); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(total, forecastedJobs, memoryStats, recordStats, runtimeStats, statusCounts); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object obj) { | ||
| if (obj == null) { | ||
| return false; | ||
| } | ||
|
|
||
| if (getClass() != obj.getClass()) { | ||
| return false; | ||
| } | ||
|
|
||
| ForecastStats other = (ForecastStats) obj; | ||
| return Objects.equals(total, other.total) && Objects.equals(forecastedJobs, other.forecastedJobs) | ||
| && Objects.equals(memoryStats, other.memoryStats) && Objects.equals(recordStats, other.recordStats) | ||
| && Objects.equals(runtimeStats, other.runtimeStats) && Objects.equals(statusCounts, other.statusCounts); | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Also add this to the
JobStats(StreamInput),writeTo(StreamOutput)with BWC checks andhashCode&equals