From d38774b63a4f7d065675c9692fa71876a03941ee Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Thu, 6 Nov 2025 21:49:20 -1000 Subject: [PATCH 1/6] [ES|QL] TS Disallow renaming timestamp prior to implicit use --- muted-tests.yml | 3 --- .../xpack/esql/qa/rest/generative/GenerativeRestTest.java | 1 + .../rules/logical/TranslateTimeSeriesAggregate.java | 6 ++++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index f754a00797d4f..8959673634304 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -464,9 +464,6 @@ tests: - class: org.elasticsearch.xpack.inference.integration.CCMStorageServiceIT method: testStoreAndGetCCMModel issue: https://github.com/elastic/elasticsearch/issues/137630 -- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeMetricsIT - method: test - issue: https://github.com/elastic/elasticsearch/issues/137655 - class: org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterBasicLicenseIT method: testLicenseInvalidForInference {p0=true} issue: https://github.com/elastic/elasticsearch/issues/137690 diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index dc68dbb41d1a6..a2027da0e03af 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -82,6 +82,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query "implicit time-series aggregation function .* doesn't support type .*", "INLINE STATS .* can only be used after STATS when used with TS command", "cannot group by a metric field .* in a time-series aggregation", + "@timestamp field has been modified", "Output has changed from \\[.*\\] to \\[.*\\]" // https://github.com/elastic/elasticsearch/issues/134794 ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java index dbdaa2aab2df7..3e2e8ac00bfd4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Holder; +import org.elasticsearch.xpack.esql.expression.function.TimestampAware; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.DimensionValues; import org.elasticsearch.xpack.esql.expression.function.aggregate.LastOverTime; @@ -240,6 +241,11 @@ protected LogicalPlan rule(TimeSeriesAggregate aggregate, LogicalOptimizerContex } } } + if (aggregate.child().output().contains(timestamp.get()) == false + && timeSeriesAggs.keySet().stream().anyMatch(ts -> ts instanceof TimestampAware)) { + // TODO: rephrase + throw new IllegalArgumentException("@timestamp field has been modified"); + } // time-series aggregates must be grouped by _tsid (and time-bucket) first and re-group by users key List firstPassGroupings = new ArrayList<>(); firstPassGroupings.add(tsid.get()); From 97e53c0e64124fa67e1b7319b4e76b3e74e2ae74 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Sun, 9 Nov 2025 16:51:40 -1000 Subject: [PATCH 2/6] add test and rephrase --- .../logical/TranslateTimeSeriesAggregate.java | 21 +++++++++--- .../optimizer/LogicalPlanOptimizerTests.java | 33 +++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java index 3e2e8ac00bfd4..cdd3ba93ea428 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java @@ -241,10 +241,23 @@ protected LogicalPlan rule(TimeSeriesAggregate aggregate, LogicalOptimizerContex } } } - if (aggregate.child().output().contains(timestamp.get()) == false - && timeSeriesAggs.keySet().stream().anyMatch(ts -> ts instanceof TimestampAware)) { - // TODO: rephrase - throw new IllegalArgumentException("@timestamp field has been modified"); + if (aggregate.child().output().contains(timestamp.get()) == false) { + var timestampAwareFunctions = timeSeriesAggs.keySet() + .stream() + .filter(ts -> ts instanceof TimestampAware) + .map(ts -> ts.sourceText()) + .toList(); + if (timestampAwareFunctions.isEmpty() == false) { + int size = timestampAwareFunctions.size(); + throw new IllegalArgumentException( + "Function" + + (size > 1 ? "s " : " ") + + "[" + + String.join(", ", timestampAwareFunctions.subList(0, Math.min(size, 3))) + + (size > 3 ? ", ..." : "") + + "] requires a @timestamp field of type date to be present when run with the TS command, but it was not present." + ); + } } // time-series aggregates must be grouped by _tsid (and time-bucket) first and re-group by users key List firstPassGroupings = new ArrayList<>(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 87942f4f25f00..2534b2764b59f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -9618,4 +9618,37 @@ public void testForkInSubquery() { final String header = "Found 1 problem\nline "; assertEquals("3:24: FORK inside subquery is not supported", e.getMessage().substring(header.length())); } + + /* + * Renaming or shadowing the @timestamp field prior to running stats with TS command is not allowed. + */ + public void testTranslateMetricsAfterRenamingTimestamp() { + assertThat( + expectThrows( + IllegalArgumentException.class, + () -> logicalOptimizerWithLatestVersion.optimize(metricsAnalyzer.analyze(parser.createStatement(""" + TS k8s | + EVAL @timestamp = region | + STATS max(network.cost), count(network.eth0.rx) + """))) + ).getMessage(), + containsString(""" + Functions [max(network.cost), count(network.eth0.rx)] requires a @timestamp field of type date \ + to be present when run with the TS command, but it was not present.""") + ); + + assertThat( + expectThrows( + IllegalArgumentException.class, + () -> logicalOptimizerWithLatestVersion.optimize(metricsAnalyzer.analyze(parser.createStatement(""" + TS k8s | + DISSECT event "%{@timestamp} %{network.total_bytes_in}" | + STATS ohxqxpSqEZ = avg(network.eth0.currently_connected_clients) + """))) + ).getMessage(), + containsString(""" + Function [avg(network.eth0.currently_connected_clients)] requires a @timestamp field of type date \ + to be present when run with the TS command, but it was not present.""") + ); + } } From 0eb4a5bde03a9d3056df9f7b00d7e167608ab617 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Sun, 9 Nov 2025 17:13:14 -1000 Subject: [PATCH 3/6] add timestamp to ToString of TimestampAware functions --- .../xpack/esql/expression/function/aggregate/Delta.java | 2 +- .../xpack/esql/expression/function/aggregate/FirstOverTime.java | 2 +- .../xpack/esql/expression/function/aggregate/Idelta.java | 2 +- .../xpack/esql/expression/function/aggregate/Increase.java | 2 +- .../xpack/esql/expression/function/aggregate/Irate.java | 2 +- .../xpack/esql/expression/function/aggregate/LastOverTime.java | 2 +- .../xpack/esql/expression/function/aggregate/Rate.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Delta.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Delta.java index 19636e5845a04..c9e28bf67e0b3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Delta.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Delta.java @@ -123,7 +123,7 @@ public Delta perTimeSeriesAggregation() { @Override public String toString() { - return "delta(" + field() + ")"; + return "delta(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java index 67ab4de0dc17a..70b0a2d7a3c11 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java @@ -141,7 +141,7 @@ public FirstOverTime perTimeSeriesAggregation() { @Override public String toString() { - return "first_over_time(" + field() + ")"; + return "first_over_time(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Idelta.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Idelta.java index c70a1b4487b76..85194ae0a9d04 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Idelta.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Idelta.java @@ -125,7 +125,7 @@ public Idelta perTimeSeriesAggregation() { @Override public String toString() { - return "idelta(" + field() + ")"; + return "idelta(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Increase.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Increase.java index 543ecf97327d7..8fcc29b98a443 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Increase.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Increase.java @@ -126,7 +126,7 @@ public Increase perTimeSeriesAggregation() { @Override public String toString() { - return "increase(" + field() + ")"; + return "increase(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Irate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Irate.java index 74bc8082beb9d..e4f003d4b9310 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Irate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Irate.java @@ -122,7 +122,7 @@ public Irate perTimeSeriesAggregation() { @Override public String toString() { - return "irate(" + field() + ")"; + return "irate(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java index 4469a44242ecd..202b9d236fe58 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java @@ -143,7 +143,7 @@ public LastOverTime perTimeSeriesAggregation() { @Override public String toString() { - return "last_over_time(" + field() + ")"; + return "last_over_time(" + field() + ", " + timestamp() + ")"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java index 48880199f188b..4edf74bc30a61 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java @@ -126,7 +126,7 @@ public Rate perTimeSeriesAggregation() { @Override public String toString() { - return "rate(" + field() + ")"; + return "rate(" + field() + ", " + timestamp() + ")"; } @Override From 27c236becbce6f65a612f1d46a40568411c0e807 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Sun, 9 Nov 2025 17:18:23 -1000 Subject: [PATCH 4/6] fix generative test message --- .../xpack/esql/qa/rest/generative/GenerativeRestTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 28fcb066de3eb..1c3926d64ed2a 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -82,7 +82,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query "implicit time-series aggregation function .* doesn't support type .*", "INLINE STATS .* can only be used after STATS when used with TS command", "cannot group by a metric field .* in a time-series aggregation", - "@timestamp field has been modified", + "requires a @timestamp field of type date to be present when run with the TS command", "Output has changed from \\[.*\\] to \\[.*\\]" // https://github.com/elastic/elasticsearch/issues/134794 ); From 446cffdaf369bdf7644f9ea1248e65674c1b5982 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Mon, 10 Nov 2025 18:23:29 -1000 Subject: [PATCH 5/6] fix test --- .../optimizer/rules/logical/TranslateTimeSeriesAggregate.java | 4 +++- .../xpack/esql/optimizer/LogicalPlanOptimizerTests.java | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java index cdd3ba93ea428..2c251981f659d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TranslateTimeSeriesAggregate.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Holder; @@ -245,7 +246,8 @@ protected LogicalPlan rule(TimeSeriesAggregate aggregate, LogicalOptimizerContex var timestampAwareFunctions = timeSeriesAggs.keySet() .stream() .filter(ts -> ts instanceof TimestampAware) - .map(ts -> ts.sourceText()) + .map(Node::sourceText) + .sorted() .toList(); if (timestampAwareFunctions.isEmpty() == false) { int size = timestampAwareFunctions.size(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 305e0f5db258e..b375d28985904 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -9675,7 +9675,6 @@ public void testFullTextFunctionOnEvalNull() { assertEquals("test", relation.indexPattern()); } - /* * Renaming or shadowing the @timestamp field prior to running stats with TS command is not allowed. */ @@ -9690,7 +9689,7 @@ STATS max(network.cost), count(network.eth0.rx) """))) ).getMessage(), containsString(""" - Functions [max(network.cost), count(network.eth0.rx)] requires a @timestamp field of type date \ + Functions [count(network.eth0.rx), max(network.cost)] requires a @timestamp field of type date \ to be present when run with the TS command, but it was not present.""") ); From 473184b002c149db9089d9f393a8747cf67f6119 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Tue, 11 Nov 2025 11:13:08 -1000 Subject: [PATCH 6/6] Update docs/changelog/137713.yaml --- docs/changelog/137713.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/137713.yaml diff --git a/docs/changelog/137713.yaml b/docs/changelog/137713.yaml new file mode 100644 index 0000000000000..7dffe59f45f58 --- /dev/null +++ b/docs/changelog/137713.yaml @@ -0,0 +1,6 @@ +pr: 137713 +summary: TS Disallow renaming into timestamp prior to implicit use +area: ES|QL +type: bug +issues: + - 137655